filesystem.py 28.9 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
7
from mimetypes import guess_type
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
10
import ConfigParser
11
import json
12
13
14
15
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
16
import stat
17
import subprocess
18
19
import fsmonitor
import vispa
20
21
22

try:
    import Image
Gero Müller's avatar
Gero Müller committed
23
    import ImageFilter
24
25
26
27
28
29
30
31
32
33
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
34

35
36
logger = logging.getLogger(__name__)

37
38
class FileSystem(object):

39
40
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
41
42
43
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
44
        "root": "text/plain"
Marcel's avatar
Marcel committed
45
    }
46
47
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
48

49
    def __init__(self, userid, workspaceid):
50
51
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
52
        self.watchservice = WatchService()
53
54
        self._userid = userid
        self._workspaceid = workspaceid
asseldonk's avatar
asseldonk committed
55
    
56
57
    def __del__(self):
        self.close()
58

59
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
60
61
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
62
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
63
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
64
65
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
66
67
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
68
        else:
Gero Müller's avatar
Gero Müller committed
69
            os.makedirs(self.basedir, 0o700)
70
            return "Basedir now exists"
71

72
    def close(self):
73
74
        if self.watchservice:
            self.watchservice.stop()
asseldonk's avatar
asseldonk committed
75
    
murban's avatar
murban committed
76
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
77
        filepath = self.expand(filepath)
78
79
80
81
        mime, encoding = guess_type(filepath)
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
82
83
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
84
85
86
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
87
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
88
        path = self.expand(path)
89
90
91
92
93
94
95
96
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
97
98
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
99
        path = self.expand(path)
100
101
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
102
103
            return None
        # type correct?
104
105
106
107
108
109
110
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
111

112
113
114
115
116
117
118
    def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
119
120
121
122
123
124
        path = self.expand(path)
        if os.path.exists(path):
            return len(os.listdir(path))
        else:
            return -1

Gero Müller's avatar
Gero Müller committed
125
126
    def get_file_list(self, path, deep=False,
                      filter=None, reverse=False,
127
128
129
130
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
131
            if self.watch(path, window_id, view_id, watch_id, filter, reverse):
132
133
134
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
135
        filter = re.compile(filter) if filter else None
136
        filelist = []
Marcel's avatar
Marcel committed
137
        path_expand = self.expand(path)
138
139
140
141
142
143
144
        try:
            for elem in os.listdir(path_expand):
                # hide hidden files?
                if elem.startswith('.') and hide_hidden:
                    continue

                # excluded by filters?
145
                if filter and bool(filter.search(elem)) != reverse:
146
                    continue
asseldonk's avatar
asseldonk committed
147
                
148
                root, ext = os.path.splitext(elem)
Gero Müller's avatar
Gero Müller committed
149
                info = {
150
151
152
                    'name': elem,
                    'root': root,
                    'ext': ext
Gero Müller's avatar
Gero Müller committed
153
                }
154

Gero Müller's avatar
Gero Müller committed
155
                fullpath = os.path.join(path_expand, elem)
Gero Müller's avatar
Gero Müller committed
156
                stats = os.lstat(fullpath)
Gero Müller's avatar
Gero Müller committed
157
                is_symlink = stat.S_ISLNK(stats.st_mode)
Gero Müller's avatar
Gero Müller committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
                if is_symlink:
                    realfullpath = os.path.realpath(fullpath)
                    info.update({'symlink': True, 'realpath': realfullpath})
                    if os.path.exists(realfullpath):
                        stats = os.stat(realfullpath)

                info.update({
                    'size': stats.st_size,
                    'mtime': stats.st_mtime,
                    'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
                })

                filelist.append(info)
                if deep:
                    filelist.extend(self.get_file_list(fullpath, deep,
                                                       filter, reverse))
        except:
            pass
176
177

        # Determine the parent
Gero Müller's avatar
Gero Müller committed
178
        # parentpath = path_expand[:-1] if path_expand.endswith(os.sep) and
Gero Müller's avatar
Gero Müller committed
179
        #    path_expand != os.sep else path_expand
180
        parentpath = os.path.dirname(path_expand)
181
182
183
        data = {'filelist': filelist, 'parentpath': parentpath}
        if encode_json:
            return json.dumps(data)
184
185
        else:
            return data
186

Gero Müller's avatar
Gero Müller committed
187
188
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
189
190
191
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
192
193
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
194
            # dir case
Marcel's avatar
Marcel committed
195
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
196
197
198
199
200
201
202
203
204
205
206
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
207
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
208
209
210
211
212
213
214
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
215
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
216
217
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
218
219
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
220
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
221
222
223
224
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
225
226
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
227
228
229
                suggestion += '/'
            suggestions.append(suggestion)

230
        return suggestions if not encode_json else json.dumps(suggestions)
231

murban's avatar
murban committed
232
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
233
        path = self.expand(path)
murban's avatar
murban committed
234
        path = path[1:] if path.startswith(os.sep) else path
235
236
        if path == "":
            return path
murban's avatar
murban committed
237
        path = path[:-1] if path.endswith(os.sep) else path
238
239
        return path

murban's avatar
murban committed
240
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
241
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
242
243
        name = self.expand(name)

244
        # folder with the same name existent?
245
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
246
        fullpath = os.path.join(path, name)
247
248
249
        try:
            os.mkdir(fullpath)
        except Exception as e:
250
            # raise Exception("You don't have the permission to create this folder!")
251
252
            raise Exception(str(e))

murban's avatar
murban committed
253
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
254
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
255
256
        name = self.expand(name)

257
        # file with the same name existent?
258
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
259
        fullpath = os.path.join(path, name)
260
261
262
263
264
265
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

266
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
267
268
269
270
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

271
272
273
274
275
276
277
        try:
            if force == False:
                new_name = self.handle_file_name_collision(new_name, path)
            name = os.path.join(path, name)
            new_name = os.path.join(path, new_name)
            os.renames(name, new_name)
            return
Gero Müller's avatar
Gero Müller committed
278
        except Exception as e:
279
            return str(e)
280

murban's avatar
murban committed
281
282
    def remove(self, path):
        if isinstance(path, list):
283
284
285
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
286
287
288
289
290
            path = self.expand(path)
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)
291

292
293
294
295
296
297
    def move(self, source, destination):
        source = self.expand(source)
        destination = self.expand(destination)
        if os.path.isdir(destination):
            shutil.move(source, destination)

298
    def compress(self, path, paths, name):
Gero Müller's avatar
Gero Müller committed
299
300
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
301
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
302
303
304

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
305

Gero Müller's avatar
Gero Müller committed
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
326
327
                            ap = fullp[
                                len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
328
329
330
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
331
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
332
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
333
                    archive.write(p, ap)
334

335
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
336
        # TODO
337
338
339
340
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
341
                self.paste(path, p, cut)
342
343
            return True

344
345
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
        logger.error("")
Gero Müller's avatar
Gero Müller committed
346
347
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
348
        fulltarget = os.path.join(path, target)
349

350
351
352
353
354
355
356
357
358
359
360
361
        orig_target = fullsrc.split(os.sep)[-1]
        orig_fulltarget = os.path.join(path, orig_target)

        if fullsrc != orig_fulltarget:
            if os.path.isdir(fullsrc):
                shutil.copytree(fullsrc, fulltarget)
                if cut:
                    shutil.rmtree(fullsrc)
            else:
                shutil.copy2(fullsrc, fulltarget)
                if cut:
                    os.remove(fullsrc)
asseldonk's avatar
asseldonk committed
362

363
        else:
364
365
366
367
368
369
370
371
372
373
374
            fulltarget_temp = orig_fulltarget + "~temp"
            if os.path.isdir(fullsrc):
                shutil.copytree(fullsrc, fulltarget_temp)
                if cut:
                    shutil.rmtree(fullsrc)
                shutil.move(fulltarget_temp, orig_fulltarget)
            else:
                shutil.copy2(fullsrc, fulltarget_temp)
                if cut:
                    os.remove(fullsrc)
                shutil.move(fulltarget_temp, orig_fulltarget)
375

376
    def save_file(self, path, content, force=True, binary=False,
asseldonk's avatar
asseldonk committed
377
                   utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
378
        path = self.expand(path)
Marcel's avatar
Marcel committed
379
        # check if file already exists
murban's avatar
murban committed
380
        if os.path.exists(path) and not force:
381
382
383
384
385
386
387
388
389
390
391
392
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
asseldonk's avatar
asseldonk committed
393
394
            mtime  = 0
        
395
396
397
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
398
        else:
399
            watch_error = ""
asseldonk's avatar
asseldonk committed
400
        
401
402
        return json.dumps({
            "mtime": os.path.getmtime(path),
asseldonk's avatar
asseldonk committed
403
404
            "success": mtime > 0 and self.checkPermissions(path),   #save is not successful, if file not writable    
            "watch_error": watch_error, 
405
            "path": path
406
        })
asseldonk's avatar
asseldonk committed
407
    
408
    def get_file(self, path, binary=False,
asseldonk's avatar
asseldonk committed
409
                  utf8=False, window_id=None, view_id=None, watch_id=None):
410
411
412
413
414
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
asseldonk's avatar
asseldonk committed
415
        
416
417
418
419
420
421
422
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
asseldonk's avatar
asseldonk committed
423
            #new: check for writing rights
424
425
            writable = self.checkPermissions(path)

426
427
            mtime = os.path.getmtime(path)
        except Exception as e:
asseldonk's avatar
asseldonk committed
428
            mtime  = 0
429
            content = ""
430
            writable = None
asseldonk's avatar
asseldonk committed
431
        
432
433
434
435
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
436
437
            "watch_error": watch_error,
            "writable": writable
438
        })
439

440
    def checkPermissions(self, path):
asseldonk's avatar
asseldonk committed
441
        return os.access(path, os.W_OK)
442

Gero Müller's avatar
Gero Müller committed
443
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
444
                          path=None, force=True, append=False):
445
446
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Gero Müller's avatar
Gero Müller committed
447
448
449
450
451
452
        #    return False, "Permission denied!"

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

453
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
454
455
456
457
458
459
460
461
462
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

murban's avatar
murban committed
463
    def get_file_content(self, path):
Marcel's avatar
Marcel committed
464
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
465
        f = open(path, "rb")
466
467
        content = f.read()
        f.close()
Marcel's avatar
Marcel committed
468
        return content
469

Marcel's avatar
Marcel committed
470
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
471
        path = self.expand(path)
Marcel's avatar
Marcel committed
472
473
        return os.path.getmtime(path)

murban's avatar
murban committed
474
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
475
        path = self.expand(path)
476
        extension = path.split(".")[-1]
477
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
478

murban's avatar
murban committed
479
    def handle_file_name_collision(self, name, path):
480
        # collision?
murban's avatar
murban committed
481
        files = os.listdir(path)
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
        counter = prename.split("_")[-1]

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
                preprename = "_".join(prename.split("_")[:-1])
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
Gero Müller's avatar
Gero Müller committed
512
513
514
            newname = "%s_%d%s" % (preprename,
                                   counter,
                                   "" if extension == "" else "." + extension)
515
        else:
Gero Müller's avatar
Gero Müller committed
516
517
            newname = "%s_1%s" % (
                prename, "" if extension == "" else "." + extension)
518
519

        # return
murban's avatar
murban committed
520
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
521

Marcel's avatar
Marcel committed
522
523
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
524

Gero Müller's avatar
Gero Müller committed
525
    def thumbnail(self, path, width=100, height=100, sharpen=True):
526
527
528
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
529
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
530
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
531
            if sharpen:
532
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
533
            img.save(output, "JPEG")
534
535
536
537
538
539
540
541
542
543
544
545
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

546
    def watch(self, path, window_id, view_id, watch_id, pattern=None, reverse=False):
547
548
549
550
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
asseldonk's avatar
asseldonk committed
551
552
        
        self.watchservice.subscribe((window_id, view_id, watch_id), path, pattern, reverse)
553
        return ""
asseldonk's avatar
asseldonk committed
554
        
555
    def unwatch(self, window_id, view_id, watch_id=None):
556
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
557
558
        return ""

559
    def get_workspaceini(self, request, fail_on_missing=False):
560
        try:
561
            request_dict = json.loads(request)
562
            config = ConfigParser.ConfigParser()
563
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
asseldonk's avatar
asseldonk committed
564
                    self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
565
566
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
asseldonk's avatar
asseldonk committed
567
                self._watch_workspaceini() 
568
569
            else:
                mtime = -1
570
            if not isinstance(request_dict, dict):
571
                request_dict = dict.fromkeys(config.sections(), True)
572
573
574
575
576
577
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
578
                        data[section] = dict(config.items(section))
579
580
581
582
583
584
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
asseldonk's avatar
asseldonk committed
585
                                raise Exception('workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
586
                elif fail_on_missing:
asseldonk's avatar
asseldonk committed
587
                    raise Exception('workspace.ini is missing the section [%s]' % section)
588
            return json.dumps({
589
                "content": data,
590
591
                "success": True,
                "mtime": mtime
asseldonk's avatar
asseldonk committed
592
                })
593
594
        except Exception as e:
            return json.dumps({
595
596
597
                "content": "",
                "success": False,
                "error": str(e)
asseldonk's avatar
asseldonk committed
598
                })
599

600
    def set_workspaceini(self, request):
601
        try:
602
603
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
asseldonk's avatar
asseldonk committed
604
                raise Exception('Given values to be set in workspace.ini in wrong format')
605
            config = ConfigParser.ConfigParser()
606
            config.read(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF))
607
608
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
asseldonk's avatar
asseldonk committed
609
                    raise Exception('Given values to be set in workspace.ini in wrong format')
610
611
612
613
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
614
615
            with open(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF), 'w') as f:
                self._watch_workspaceini()
616
617
                config.write(f)
            return ""
618
        except Exception as e:
619
            return str(e)
asseldonk's avatar
asseldonk committed
620

621
    def _watch_workspaceini(self):
622
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
asseldonk's avatar
asseldonk committed
623
            self.watchservice.subscribe((self._userid, self._workspaceid), FileSystem.PRIVATE_WORKSPACE_CONF)
624
625
626
627
628
629
630
631
632
633
634

class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
        self.thread.start()
asseldonk's avatar
asseldonk committed
635
    
636
637
638
    def subscribe(self, id, path, pattern=None, reverse=False):
        if not path:
            return self.unsubscribe(id)
asseldonk's avatar
asseldonk committed
639
        
640
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
asseldonk's avatar
asseldonk committed
641
        
642
643
644
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
asseldonk's avatar
asseldonk committed
645
            
646
            self.subscribers[id].update(path, pattern, reverse)
asseldonk's avatar
asseldonk committed
647
    
648
649
650
651
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
asseldonk's avatar
asseldonk committed
652
                    if False not in map(lambda e,c: c is None or e == c, subscriber.id, id):
653
654
655
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
asseldonk's avatar
asseldonk committed
656
    
657
658
    def stop(self):
        self.run = False
asseldonk's avatar
asseldonk committed
659
    
660
661
662
663
664
665
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
asseldonk's avatar
asseldonk committed
666
                        if event.action_name in ['delete self','move self']:
667
668
669
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
asseldonk's avatar
asseldonk committed
670
                        elif event.watch.isdir and event.action_name in ['create','delete','move from','move to']:
671
672
673
674
675
676
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
asseldonk's avatar
asseldonk committed
677
                                    event.watch.mtime = os.path.getmtime(event.watch.path)
678
679
680
681
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
asseldonk's avatar
asseldonk committed
682
            
683
684
685
686
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
asseldonk's avatar
asseldonk committed
687
        
688
        for subscriber in self.subscribers.items():
689
            subscriber.destroy()
asseldonk's avatar
asseldonk committed
690
        
691
692
        self.monitor.remove_all_watches()
        self.monitor.close()
asseldonk's avatar
asseldonk committed
693
694
    
class WatchSubscriber(object): # this should never be instanced manually
695
    EVENT_DELAYS = {
asseldonk's avatar
asseldonk committed
696
697
        'change': [1.0,0.1],
        'modify': [1.0,0.2]
698
    }
asseldonk's avatar
asseldonk committed
699
    
700
701
702
703
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
asseldonk's avatar
asseldonk committed
704
            raise RuntimeError("There is already a subscriber with this id: "+str(id))
705
706
707
708
709
710
711
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
        self.event_buffer = {}
asseldonk's avatar
asseldonk committed
712
    
713
714
715
716
717
718
719
720
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
asseldonk's avatar
asseldonk committed
721
    
722
    def process(self, event, subject=""):
723
        if self.watch.isdir and subject and self.pattern and bool(self.pattern.search(subject)) != self.reverse:
724
            return
asseldonk's avatar
asseldonk committed
725
        
726
727
728
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
asseldonk's avatar
asseldonk committed
729
                self.event_buffer[event][1] = now + WatchSubscriber.EVENT_DELAYS[event][1]
730
            else:
asseldonk's avatar
asseldonk committed
731
                self.event_buffer[event] = [now + delay for delay in WatchSubscriber.EVENT_DELAYS[event]] #first & last event
732
733
734
735
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
asseldonk's avatar
asseldonk committed
736
    
737
738
739
740
741
742
743
744
745
746
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
asseldonk's avatar
asseldonk committed
747
        if len(self.id) == 3: # window_id, view_id, watch_id
748
749
750
751
752
753
754
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
            if not self.watch.isdir:
                data['mtime'] = self.watch.mtime
asseldonk's avatar
asseldonk committed
755
756
            vispa.remote.send_topic("extension.%s.socket.watch" % self.id[1], window_id=self.id[0], data=data)
        elif len(self.id) == 2: # userid, workspaceid
757
758
759
760
761
762
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
asseldonk's avatar
asseldonk committed
763
    
764
765
766
767
768
769
770
771
772
    def update(self, path, pattern="", reverse=False):
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
        else:
            self.pattern = None
            self.reverse = None
asseldonk's avatar
asseldonk committed
773
    
774
775
776
777
778
779
    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()
asseldonk's avatar
asseldonk committed
780
        
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]
asseldonk's avatar
asseldonk committed
797
        
798
799
800
        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)
asseldonk's avatar
asseldonk committed
801
    
802
803
    def unbind(self):
        if not self.watch:
804
            return
asseldonk's avatar
asseldonk committed
805
        
806
807
808
809
        self.watch.subscribers.remove(self)
        if len(self.watch.subscribers) == 0:
            del self.service.watches[self.watch.path]
            self.service.monitor.remove_watch(self.watch)
asseldonk's avatar
asseldonk committed
810
        
811
        self.watch = None
812

Marcel's avatar
Marcel committed
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1

def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1