Aufgrund einer Wartung wird GitLab am 18.01. zwischen 8:00 und 9:00 Uhr kurzzeitig nicht zur Verfügung stehen. / Due to maintenance, GitLab will be temporarily unavailable on 18.01. between 8:00 and 9:00 am.

filesystem.py 31.5 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
import mimetypes
7
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
10
import ConfigParser
11
import json
12
13
14
15
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
16
import stat
17
import subprocess
18
19
import fsmonitor
import vispa
20
21
22

try:
    import Image
Gero Müller's avatar
Gero Müller committed
23
    import ImageFilter
Martin Urban's avatar
Martin Urban committed
24

25
26
27
28
29
30
31
32
33
34
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
35

36
37
logger = logging.getLogger(__name__)

Martin Urban's avatar
Martin Urban committed
38

39
40
41
42
43
44
45
46
47
48
49
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
Martin Urban's avatar
Martin Urban committed
50

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

asseldonk's avatar
asseldonk committed
70

Martin Urban's avatar
Martin Urban committed
71
class FileSystem(object):
72
73
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
74
75
76
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
77
        "root": "text/plain"
Marcel's avatar
Marcel committed
78
    }
79
80
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
81

82
    def __init__(self, userid, workspaceid):
83
84
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
85
        self.watchservice = WatchService()
86
87
        self._userid = userid
        self._workspaceid = workspaceid
88
        mimetypes.init()
Martin Urban's avatar
Martin Urban committed
89

90
91
    def __del__(self):
        self.close()
92

93
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
94
95
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
96
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
97
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
98
99
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
100
101
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
102
        else:
Gero Müller's avatar
Gero Müller committed
103
            os.makedirs(self.basedir, 0o700)
104
            return "Basedir now exists"
105

106
    def close(self):
107
108
        if self.watchservice:
            self.watchservice.stop()
Martin Urban's avatar
Martin Urban committed
109

murban's avatar
murban committed
110
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
111
        filepath = self.expand(filepath)
112
        mime, encoding = mimetypes.guess_type(filepath)
113
114
115
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
116
117
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
118
119
120
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
121
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
122
        path = self.expand(path)
123
124
125
126
127
128
129
130
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
131
132
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
133
        path = self.expand(path)
134
135
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
136
137
            return None
        # type correct?
138
139
140
141
142
143
144
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
145

146
147
    def get_file_count(
            self, path, window_id=None, view_id=None, watch_id=None):
148
149
150
151
152
153
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
154
155
        path = self.expand(path)
        if os.path.exists(path):
Martin Urban's avatar
Martin Urban committed
156
157
158
159
160
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
161
162
163
        else:
            return -1

164
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
165
                      filter=None, reverse=False,
166
167
168
169
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
170
171
            if self.watch(
                    path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
172
173
174
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
175
        filter = re.compile(filter) if filter else None
176
        base = self.expand(path)
Martin Urban's avatar
Martin Urban committed
177

178
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
Martin Urban's avatar
Martin Urban committed
179
180
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
181
182
        # ignore failed file info (e.g. access error)
        filelist = [i for i in filelist if 'size' in i]
183
184

        # Determine the parent
185
186
187
188
189
190
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
191
192
        if encode_json:
            return json.dumps(data)
193
194
        else:
            return data
Martin Urban's avatar
Martin Urban committed
195

Gero Müller's avatar
Gero Müller committed
196
197
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
198
199
200
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
201
202
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
203
            # dir case
Marcel's avatar
Marcel committed
204
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
205
206
207
208
209
210
211
212
213
214
215
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
216
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
217
218
219
220
221
222
223
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
224
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
225
226
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
227
228
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
229
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
230
231
232
233
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
234
235
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
236
237
238
                suggestion += '/'
            suggestions.append(suggestion)

239
        return suggestions if not encode_json else json.dumps(suggestions)
240

murban's avatar
murban committed
241
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
242
        path = self.expand(path)
murban's avatar
murban committed
243
        path = path[1:] if path.startswith(os.sep) else path
244
245
        if path == "":
            return path
murban's avatar
murban committed
246
        path = path[:-1] if path.endswith(os.sep) else path
247
248
        return path

murban's avatar
murban committed
249
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
250
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
251
252
        name = self.expand(name)

253
        # folder with the same name existent?
254
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
255
        fullpath = os.path.join(path, name)
256
257
258
        try:
            os.mkdir(fullpath)
        except Exception as e:
259
            # raise Exception("You don't have the permission to create this folder!")
260
261
            raise Exception(str(e))

murban's avatar
murban committed
262
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
263
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
264
265
        name = self.expand(name)

266
        # file with the same name existent?
267
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
268
        fullpath = os.path.join(path, name)
269
270
271
272
273
274
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

275
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
276
277
278
279
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

280
281
282
283
284
        if force == False:
            new_name = self.handle_file_name_collision(new_name, path)
        name = os.path.join(path, name)
        new_name = os.path.join(path, new_name)
        os.renames(name, new_name)
285

murban's avatar
murban committed
286
287
    def remove(self, path):
        if isinstance(path, list):
288
289
290
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
291
            path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
292
293
294
            if os.path.islink(path):
                os.unlink(path)
            elif os.path.isdir(path):
Marcel's avatar
Marcel committed
295
296
297
                shutil.rmtree(path)
            else:
                os.remove(path)
298

299
    def move(self, source, destination):
300
301
302
303
304
305
        if isinstance(source, list):
            for s in source:
                self.move(s, destination)
        else:
            source = self.expand(source)
            destination = self.expand(destination)
306
307
308
309
            name = os.path.split(source)[1]
            newname = self.handle_file_name_collision(name, destination)
            destination = os.path.join(destination, newname)
            if os.path.isdir(source):
310
                shutil.copytree(source, destination, symlinks=True)
311
312
313
314
315
                shutil.rmtree(source)
            else:
                shutil.copy2(source, destination)
                os.remove(source)

316

317
    def compress(self, path, paths, name):
Gero Müller's avatar
Gero Müller committed
318
319
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
320
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
321
322
323

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
324

Gero Müller's avatar
Gero Müller committed
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
345
                            ap = fullp[
346
                                 len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
347
348
349
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
350
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
351
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
352
                    archive.write(p, ap)
353

354
    def decompress(self, path):
355
        # filepath and extract path
356
357
358
359
        path = self.expand(path)

        # "foo/bar/file.zip" -> ("foo/bar", "file")
        dst = os.path.split(os.path.splitext(path)[0])
360

361
362
363
        with ZipFile(path, "r") as archive:
            dstdir = os.path.join(dst[0], self.handle_file_name_collision(dst[1], dst[0]))
            archive.extractall(dstdir)
364

365
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
366
        # TODO
367
368
369
370
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
371
                self.paste(path, p, cut)
372
373
            return True

374
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
Gero Müller's avatar
Gero Müller committed
375
376
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
377
        fulltarget = os.path.join(path, target)
378

asseldonk's avatar
asseldonk committed
379
        if os.path.isdir(fullsrc):
380
            shutil.copytree(fullsrc, fulltarget, symlinks=True)
asseldonk's avatar
asseldonk committed
381
382
            if cut:
                shutil.rmtree(fullsrc)
383
        else:
asseldonk's avatar
asseldonk committed
384
385
386
            shutil.copy2(fullsrc, fulltarget)
            if cut:
                os.remove(fullsrc)
387

388
    def save_file(self, path, content, force=True, binary=False,
389
                  utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
390
        path = self.expand(path)
Marcel's avatar
Marcel committed
391
        # check if file already exists
murban's avatar
murban committed
392
        if os.path.exists(path) and not force:
393
394
395
396
397
398
399
400
401
402
403
404
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
405
406
            mtime = 0

407
408
409
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
410
        else:
411
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
412

413
414
        return json.dumps({
            "mtime": os.path.getmtime(path),
415
416
            # save is not successful, if file not writable
            "success": mtime > 0 and self.checkPermissions(path),
Martin Urban's avatar
Martin Urban committed
417
            "watch_error": watch_error,
418
            "path": path
419
        })
Martin Urban's avatar
Martin Urban committed
420

421
    def get_file(self, path, binary=False,
Martin Urban's avatar
Martin Urban committed
422
                 utf8=False, window_id=None, view_id=None, watch_id=None):
423
424
425
426
427
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
428

429
430
431
432
433
434
435
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
Martin Urban's avatar
Martin Urban committed
436
            # new: check for writing rights
437
            writable = self.checkPermissions(path)
438
            error = None
439

440
441
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
442
            mtime = 0
443
            content = ""
444
            writable = None
445
            error = str(e)
Martin Urban's avatar
Martin Urban committed
446

447
448
449
450
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
451
            "watch_error": watch_error,
Martin Urban's avatar
Martin Urban committed
452
            "writable": writable,
453
            "error": error
454
        })
455

Martin Urban's avatar
Martin Urban committed
456
457
    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)
458

Gero Müller's avatar
Gero Müller committed
459
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
460
                          path=None, force=True, append=False):
461
462
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Martin Urban's avatar
Martin Urban committed
463
        # return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
464
465
466
467
468

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

469
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
470
471
472
473
474
475
476
477
478
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

479
480
481
482
    def get_file_content(self, path, offset=0, length=None):
        with open(self.expand(path), "rb") as f:
            f.seek(offset)
            content = f.read() if length is None else f.read(length)
Marcel's avatar
Marcel committed
483
        return content
484

Marcel's avatar
Marcel committed
485
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
486
        path = self.expand(path)
Marcel's avatar
Marcel committed
487
488
        return os.path.getmtime(path)

489
490
491
492
    def stat(self, path):
        path = self.expand(path)
        return os.stat(path)

murban's avatar
murban committed
493
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
494
        path = self.expand(path)
495
        extension = path.split(".")[-1]
496
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
497

murban's avatar
murban committed
498
    def handle_file_name_collision(self, name, path):
499
        # collision?
500
        path = self.expand(path)
murban's avatar
murban committed
501
        files = os.listdir(path)
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
519
        counter = prename.split("_copy")[-1]
520
521
522
523
524

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
525
                preprename = "_copy".join(prename.split("_copy")[:-1])
526
527
528
529
530
531
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
532
            newname = "%s_copy%d%s" % (preprename,
Gero Müller's avatar
Gero Müller committed
533
534
                                   counter,
                                   "" if extension == "" else "." + extension)
535
        else:
536
            newname = "%s_copy1%s" % (
Gero Müller's avatar
Gero Müller committed
537
                prename, "" if extension == "" else "." + extension)
538
539

        # return
murban's avatar
murban committed
540
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
541

Marcel's avatar
Marcel committed
542
543
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
544

Gero Müller's avatar
Gero Müller committed
545
    def thumbnail(self, path, width=100, height=100, sharpen=True):
546
547
548
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
549
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
550
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
551
            if sharpen:
552
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
553
            img.save(output, "JPEG")
554
555
556
557
558
559
560
561
562
563
564
565
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

566
567
    def watch(self, path, window_id, view_id, watch_id,
              pattern=None, reverse=False, hide_hidden=True):
568
569
570
571
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
Martin Urban's avatar
Martin Urban committed
572
573
574
575
576

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

577
578
        #first: remove the old watch
        self.unwatch(window_id, view_id, watch_id)
579

580
581
582
583
584
585
586
587
        self.watchservice.subscribe(
            (window_id,
             view_id,
             watch_id),
            path,
            pattern,
            reverse,
            hide_hidden)
588
        return ""
Martin Urban's avatar
Martin Urban committed
589

590
    def unwatch(self, window_id, view_id, watch_id=None):
591
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
592
593
        return ""

594
    def get_workspaceini(self, request, fail_on_missing=False):
595
        try:
596
            request_dict = json.loads(request)
597
            config = ConfigParser.ConfigParser()
598
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
Martin Urban's avatar
Martin Urban committed
599
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
600
601
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
Martin Urban's avatar
Martin Urban committed
602
                self._watch_workspaceini()
603
604
            else:
                mtime = -1
605
            if not isinstance(request_dict, dict):
606
                request_dict = dict.fromkeys(config.sections(), True)
607
608
609
610
611
612
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
613
                        data[section] = dict(config.items(section))
614
615
616
617
618
619
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
Martin Urban's avatar
Martin Urban committed
620
621
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
622
                elif fail_on_missing:
623
624
625
                    raise Exception(
                        'workspace.ini is missing the section [%s]' %
                        section)
626
            return json.dumps({
627
                "content": data,
628
629
                "success": True,
                "mtime": mtime
Martin Urban's avatar
Martin Urban committed
630
            })
631
632
        except Exception as e:
            return json.dumps({
633
634
635
                "content": "",
                "success": False,
                "error": str(e)
Martin Urban's avatar
Martin Urban committed
636
            })
637

638
    def set_workspaceini(self, request):
639
        try:
640
641
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
642
643
                raise Exception(
                    'Given values to be set in workspace.ini in wrong format')
644
            filename = self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)
645
            config = ConfigParser.ConfigParser()
646
            config.read(filename)
647
648
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
649
650
                    raise Exception(
                        'Given values to be set in workspace.ini in wrong format')
651
652
653
654
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
655
656
657
658
            filedir = os.path.dirname(filename)
            if not os.path.isdir(filedir):
                os.makedirs(filedir)
            with open(filename, 'w') as f:
659
                config.write(f)
660
            self._watch_workspaceini()
661
            return ""
662
        except Exception as e:
663
            return str(e)
asseldonk's avatar
asseldonk committed
664

665
    def _watch_workspaceini(self):
666
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
667
668
669
670
            self.watchservice.subscribe(
                (self._userid,
                 self._workspaceid),
                FileSystem.PRIVATE_WORKSPACE_CONF)
671

Martin Urban's avatar
Martin Urban committed
672

673
674
675
676
677
678
679
680
681
class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
682
        self.thread.daemon = True
683
        self.thread.start()
Martin Urban's avatar
Martin Urban committed
684

685
686
    def subscribe(
            self, id, path, pattern=None, reverse=False, hide_hidden=True):
687
688
        if not path:
            return self.unsubscribe(id)
Martin Urban's avatar
Martin Urban committed
689

690
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
Martin Urban's avatar
Martin Urban committed
691

692
693
694
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
Martin Urban's avatar
Martin Urban committed
695

696
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
Martin Urban's avatar
Martin Urban committed
697

698
699
700
701
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
702
703
                    if False not in map(
                            lambda e, c: c is None or e == c, subscriber.id, id):
704
705
706
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
Martin Urban's avatar
Martin Urban committed
707

708
709
    def stop(self):
        self.run = False
Martin Urban's avatar
Martin Urban committed
710

711
712
713
714
715
716
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
Martin Urban's avatar
Martin Urban committed
717
                        if event.action_name in ['delete self', 'move self']:
718
719
720
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
Martin Urban's avatar
Martin Urban committed
721
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
722
723
724
725
726
727
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
728
729
                                    event.watch.mtime = os.path.getmtime(
                                        event.watch.path)
730
731
732
733
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
Martin Urban's avatar
Martin Urban committed
734

735
736
737
738
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
Martin Urban's avatar
Martin Urban committed
739

740
        for subscriber in self.subscribers.items():
741
            subscriber.destroy()
Martin Urban's avatar
Martin Urban committed
742

743
        self.monitor.remove_all_watches()
744
        del self.monitor
Martin Urban's avatar
Martin Urban committed
745
746
747


class WatchSubscriber(object):  # this should never be instanced manually
748
    EVENT_DELAYS = {
Martin Urban's avatar
Martin Urban committed
749
750
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
751
    }
752
753
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
Martin Urban's avatar
Martin Urban committed
754

755
756
757
758
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
759
760
761
            raise RuntimeError(
                "There is already a subscriber with this id: " +
                str(id))
762
763
764
765
766
767
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
768
        self.hide_hidden = None
769
        self.event_buffer = {}
770
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
771

772
773
774
775
776
777
778
779
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
780
        del self.subject_buffer
Martin Urban's avatar
Martin Urban committed
781

782
    def process(self, event, subject=""):
783
784
785
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
786
787
            if self.pattern and bool(
                    self.pattern.search(subject)) != self.reverse:
788
                return
Martin Urban's avatar
Martin Urban committed
789

790
791
792
793
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
Martin Urban's avatar
Martin Urban committed
794

795
796
797
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
798
                self.event_buffer[event][1] = now + \
799
                                              WatchSubscriber.EVENT_DELAYS[event][1]
800
            else:
Martin Urban's avatar
Martin Urban committed
801
802
                self.event_buffer[event] = [now + delay for delay in
                                            WatchSubscriber.EVENT_DELAYS[event]]  # first & last event
803
804
805
806
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
Martin Urban's avatar
Martin Urban committed
807

808
809
810
811
812
813
814
815
816
817
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
Martin Urban's avatar
Martin Urban committed
818
        if len(self.id) == 3:  # window_id, view_id, watch_id
819
820
821
822
823
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
824
825
826
827
828
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
Martin Urban's avatar
Martin Urban committed
829
830
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in
                                                 self.subject_buffer[event]]
831
832
833
834
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
835
                data['mtime'] = self.watch.mtime
836
837
838
839
840
            vispa.remote.send_topic(
                "extension.%s.socket.watch" %
                self.id[1],
                window_id=self.id[0],
                data=data)
Martin Urban's avatar
Martin Urban committed
841
        elif len(self.id) == 2:  # userid, workspaceid
842
843
844
845
846
847
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
Martin Urban's avatar
Martin Urban committed
848

849
    def update(self, path, pattern="", reverse=False, hide_hidden=True):
850
851
852
853
854
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
855
856
857
858
859
860
861
862
            old_subject_buffer = self.subject_buffer
            self.subject_buffer = {}
            for event, subjects in old_subject_buffer.items():
                new_subject_list = [
                    subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
                ]
                if len(new_subject_list):
                    self.subject_buffer[event] = new_subject_list
863
864
865
        else:
            self.pattern = None
            self.reverse = None
866
        self.hide_hidden = hide_hidden
Martin Urban's avatar
Martin Urban committed
867

868
869
870
871
872
873
    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()
Martin Urban's avatar
Martin Urban committed
874

875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]
Martin Urban's avatar
Martin Urban committed
891

892
893
894
        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)
Martin Urban's avatar
Martin Urban committed
895

896
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
897

898
899
    def unbind(self):
        if not self.watch:
900
            return
Martin Urban's avatar
Martin Urban committed
901

902
903
904
905
        self.watch.subscribers.remove(self)
        if len(self.watch.subscribers) == 0:
            del self.service.watches[self.watch.path]
            self.service.monitor.remove_watch(self.watch)
Martin Urban's avatar
Martin Urban committed
906

907
        self.watch = None
908

Martin Urban's avatar
Martin Urban committed
909

Marcel's avatar
Marcel committed
910
911
912
913
914
915
916
917
def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1

Martin Urban's avatar
Martin Urban committed
918

Marcel's avatar
Marcel committed
919
920
921
922
923
924
925
926
927
def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1