filesystem.py 31 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
7
from mimetypes import guess_type
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
10
import ConfigParser
11
import json
12
13
14
15
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
16
import stat
17
import subprocess
18
19
import fsmonitor
import vispa
20
21
22

try:
    import Image
Gero Müller's avatar
Gero Müller committed
23
    import ImageFilter
Martin Urban's avatar
Martin Urban committed
24

25
26
27
28
29
30
31
32
33
34
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
35

36
37
logger = logging.getLogger(__name__)

Martin Urban's avatar
Martin Urban committed
38

39
40
41
42
43
44
45
46
47
48
49
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
Martin Urban's avatar
Martin Urban committed
50

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

asseldonk's avatar
asseldonk committed
70

Martin Urban's avatar
Martin Urban committed
71
class FileSystem(object):
72
73
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
74
75
76
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
77
        "root": "text/plain"
Marcel's avatar
Marcel committed
78
    }
79
80
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
81

82
    def __init__(self, userid, workspaceid):
83
84
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
85
        self.watchservice = WatchService()
86
87
        self._userid = userid
        self._workspaceid = workspaceid
Martin Urban's avatar
Martin Urban committed
88

89
90
    def __del__(self):
        self.close()
91

92
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
93
94
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
95
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
96
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
97
98
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
99
100
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
101
        else:
Gero Müller's avatar
Gero Müller committed
102
            os.makedirs(self.basedir, 0o700)
103
            return "Basedir now exists"
104

105
    def close(self):
106
107
        if self.watchservice:
            self.watchservice.stop()
Martin Urban's avatar
Martin Urban committed
108

murban's avatar
murban committed
109
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
110
        filepath = self.expand(filepath)
111
112
113
114
        mime, encoding = guess_type(filepath)
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
115
116
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
117
118
119
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
120
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
121
        path = self.expand(path)
122
123
124
125
126
127
128
129
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
130
131
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
132
        path = self.expand(path)
133
134
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
135
136
            return None
        # type correct?
137
138
139
140
141
142
143
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
144

145
146
    def get_file_count(
            self, path, window_id=None, view_id=None, watch_id=None):
147
148
149
150
151
152
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
153
154
        path = self.expand(path)
        if os.path.exists(path):
Martin Urban's avatar
Martin Urban committed
155
156
157
158
159
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
160
161
162
        else:
            return -1

163
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
164
                      filter=None, reverse=False,
165
166
167
168
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
169
170
            if self.watch(
                    path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
171
172
173
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
174
        filter = re.compile(filter) if filter else None
175
        base = self.expand(path)
Martin Urban's avatar
Martin Urban committed
176

177
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
Martin Urban's avatar
Martin Urban committed
178
179
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
180
181
        # ignore failed file info (e.g. access error)
        filelist = [i for i in filelist if 'size' in i]
182
183

        # Determine the parent
184
185
186
187
188
189
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
190
191
        if encode_json:
            return json.dumps(data)
192
193
        else:
            return data
Martin Urban's avatar
Martin Urban committed
194

Gero Müller's avatar
Gero Müller committed
195
196
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
197
198
199
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
200
201
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
202
            # dir case
Marcel's avatar
Marcel committed
203
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
204
205
206
207
208
209
210
211
212
213
214
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
215
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
216
217
218
219
220
221
222
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
223
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
224
225
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
226
227
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
228
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
229
230
231
232
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
233
234
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
235
236
237
                suggestion += '/'
            suggestions.append(suggestion)

238
        return suggestions if not encode_json else json.dumps(suggestions)
239

murban's avatar
murban committed
240
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
241
        path = self.expand(path)
murban's avatar
murban committed
242
        path = path[1:] if path.startswith(os.sep) else path
243
244
        if path == "":
            return path
murban's avatar
murban committed
245
        path = path[:-1] if path.endswith(os.sep) else path
246
247
        return path

murban's avatar
murban committed
248
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
249
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
250
251
        name = self.expand(name)

252
        # folder with the same name existent?
253
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
254
        fullpath = os.path.join(path, name)
255
256
257
        try:
            os.mkdir(fullpath)
        except Exception as e:
258
            # raise Exception("You don't have the permission to create this folder!")
259
260
            raise Exception(str(e))

murban's avatar
murban committed
261
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
262
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
263
264
        name = self.expand(name)

265
        # file with the same name existent?
266
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
267
        fullpath = os.path.join(path, name)
268
269
270
271
272
273
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

274
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
275
276
277
278
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

279
280
281
282
283
284
285
        try:
            if force == False:
                new_name = self.handle_file_name_collision(new_name, path)
            name = os.path.join(path, name)
            new_name = os.path.join(path, new_name)
            os.renames(name, new_name)
            return
Gero Müller's avatar
Gero Müller committed
286
        except Exception as e:
287
            return str(e)
288

murban's avatar
murban committed
289
290
    def remove(self, path):
        if isinstance(path, list):
291
292
293
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
294
            path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
295
296
297
            if os.path.islink(path):
                os.unlink(path)
            elif os.path.isdir(path):
Marcel's avatar
Marcel committed
298
299
300
                shutil.rmtree(path)
            else:
                os.remove(path)
301

302
    def move(self, source, destination):
303
304
305
306
307
308
        if isinstance(source, list):
            for s in source:
                self.move(s, destination)
        else:
            source = self.expand(source)
            destination = self.expand(destination)
309
310
311
312
313
314
315
316
317
318
            name = os.path.split(source)[1]
            newname = self.handle_file_name_collision(name, destination)
            destination = os.path.join(destination, newname)
            if os.path.isdir(source):
                shutil.copytree(source, destination)
                shutil.rmtree(source)
            else:
                shutil.copy2(source, destination)
                os.remove(source)

319

320
    def compress(self, path, paths, name):
Gero Müller's avatar
Gero Müller committed
321
322
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
323
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
324
325
326

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
327

Gero Müller's avatar
Gero Müller committed
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
348
                            ap = fullp[
349
                                 len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
350
351
352
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
353
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
354
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
355
                    archive.write(p, ap)
356

357
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
358
        # TODO
359
360
361
362
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
363
                self.paste(path, p, cut)
364
365
            return True

366
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
Gero Müller's avatar
Gero Müller committed
367
368
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
369
        fulltarget = os.path.join(path, target)
370

asseldonk's avatar
asseldonk committed
371
372
373
374
        if os.path.isdir(fullsrc):
            shutil.copytree(fullsrc, fulltarget)
            if cut:
                shutil.rmtree(fullsrc)
375
        else:
asseldonk's avatar
asseldonk committed
376
377
378
            shutil.copy2(fullsrc, fulltarget)
            if cut:
                os.remove(fullsrc)
379

380
    def save_file(self, path, content, force=True, binary=False,
381
                  utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
382
        path = self.expand(path)
Marcel's avatar
Marcel committed
383
        # check if file already exists
murban's avatar
murban committed
384
        if os.path.exists(path) and not force:
385
386
387
388
389
390
391
392
393
394
395
396
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
397
398
            mtime = 0

399
400
401
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
402
        else:
403
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
404

405
406
        return json.dumps({
            "mtime": os.path.getmtime(path),
407
408
            # save is not successful, if file not writable
            "success": mtime > 0 and self.checkPermissions(path),
Martin Urban's avatar
Martin Urban committed
409
            "watch_error": watch_error,
410
            "path": path
411
        })
Martin Urban's avatar
Martin Urban committed
412

413
    def get_file(self, path, binary=False,
Martin Urban's avatar
Martin Urban committed
414
                 utf8=False, window_id=None, view_id=None, watch_id=None):
415
416
417
418
419
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
420

421
422
423
424
425
426
427
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
Martin Urban's avatar
Martin Urban committed
428
            # new: check for writing rights
429
            writable = self.checkPermissions(path)
430
            error = None
431

432
433
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
434
            mtime = 0
435
            content = ""
436
            writable = None
437
            error = str(e)
Martin Urban's avatar
Martin Urban committed
438

439
440
441
442
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
443
            "watch_error": watch_error,
Martin Urban's avatar
Martin Urban committed
444
            "writable": writable,
445
            "error": error
446
        })
447

Martin Urban's avatar
Martin Urban committed
448
449
    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)
450

Gero Müller's avatar
Gero Müller committed
451
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
452
                          path=None, force=True, append=False):
453
454
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Martin Urban's avatar
Martin Urban committed
455
        # return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
456
457
458
459
460

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

461
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
462
463
464
465
466
467
468
469
470
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

471
    def get_file_content(self, path, offset=0, size=None):
Marcel's avatar
Marcel committed
472
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
473
        f = open(path, "rb")
474
475
        f.seek(offset)
        content = f.read(size)
476
        f.close()
Marcel's avatar
Marcel committed
477
        return content
478

Marcel's avatar
Marcel committed
479
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
480
        path = self.expand(path)
Marcel's avatar
Marcel committed
481
482
        return os.path.getmtime(path)

483
484
485
486
    def stat(self, path):
        path = self.expand(path)
        return os.stat(path)

murban's avatar
murban committed
487
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
488
        path = self.expand(path)
489
        extension = path.split(".")[-1]
490
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
491

murban's avatar
murban committed
492
    def handle_file_name_collision(self, name, path):
493
        # collision?
494
        path = self.expand(path)
murban's avatar
murban committed
495
        files = os.listdir(path)
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
513
        counter = prename.split("_copy")[-1]
514
515
516
517
518

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
519
                preprename = "_copy".join(prename.split("_copy")[:-1])
520
521
522
523
524
525
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
526
            newname = "%s_copy%d%s" % (preprename,
Gero Müller's avatar
Gero Müller committed
527
528
                                   counter,
                                   "" if extension == "" else "." + extension)
529
        else:
530
            newname = "%s_copy1%s" % (
Gero Müller's avatar
Gero Müller committed
531
                prename, "" if extension == "" else "." + extension)
532
533

        # return
murban's avatar
murban committed
534
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
535

Marcel's avatar
Marcel committed
536
537
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
538

Gero Müller's avatar
Gero Müller committed
539
    def thumbnail(self, path, width=100, height=100, sharpen=True):
540
541
542
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
543
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
544
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
545
            if sharpen:
546
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
547
            img.save(output, "JPEG")
548
549
550
551
552
553
554
555
556
557
558
559
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

560
561
    def watch(self, path, window_id, view_id, watch_id,
              pattern=None, reverse=False, hide_hidden=True):
562
563
564
565
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
Martin Urban's avatar
Martin Urban committed
566
567
568
569
570

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

571
572
573
        #first: remove the old watch
        self.unwatch(window_id, view_id, watch_id)
        
574
575
576
577
578
579
580
581
        self.watchservice.subscribe(
            (window_id,
             view_id,
             watch_id),
            path,
            pattern,
            reverse,
            hide_hidden)
582
        return ""
Martin Urban's avatar
Martin Urban committed
583

584
    def unwatch(self, window_id, view_id, watch_id=None):
585
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
586
587
        return ""

588
    def get_workspaceini(self, request, fail_on_missing=False):
589
        try:
590
            request_dict = json.loads(request)
591
            config = ConfigParser.ConfigParser()
592
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
Martin Urban's avatar
Martin Urban committed
593
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
594
595
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
Martin Urban's avatar
Martin Urban committed
596
                self._watch_workspaceini()
597
598
            else:
                mtime = -1
599
            if not isinstance(request_dict, dict):
600
                request_dict = dict.fromkeys(config.sections(), True)
601
602
603
604
605
606
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
607
                        data[section] = dict(config.items(section))
608
609
610
611
612
613
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
Martin Urban's avatar
Martin Urban committed
614
615
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
616
                elif fail_on_missing:
617
618
619
                    raise Exception(
                        'workspace.ini is missing the section [%s]' %
                        section)
620
            return json.dumps({
621
                "content": data,
622
623
                "success": True,
                "mtime": mtime
Martin Urban's avatar
Martin Urban committed
624
            })
625
626
        except Exception as e:
            return json.dumps({
627
628
629
                "content": "",
                "success": False,
                "error": str(e)
Martin Urban's avatar
Martin Urban committed
630
            })
631

632
    def set_workspaceini(self, request):
633
        try:
634
635
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
636
637
                raise Exception(
                    'Given values to be set in workspace.ini in wrong format')
638
            filename = self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)
639
            config = ConfigParser.ConfigParser()
640
            config.read(filename)
641
642
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
643
644
                    raise Exception(
                        'Given values to be set in workspace.ini in wrong format')
645
646
647
648
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
649
650
651
652
            filedir = os.path.dirname(filename)
            if not os.path.isdir(filedir):
                os.makedirs(filedir)
            with open(filename, 'w') as f:
653
                config.write(f)
654
            self._watch_workspaceini()
655
            return ""
656
        except Exception as e:
657
            return str(e)
asseldonk's avatar
asseldonk committed
658

659
    def _watch_workspaceini(self):
660
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
661
662
663
664
            self.watchservice.subscribe(
                (self._userid,
                 self._workspaceid),
                FileSystem.PRIVATE_WORKSPACE_CONF)
665

Martin Urban's avatar
Martin Urban committed
666

667
668
669
670
671
672
673
674
675
676
class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
        self.thread.start()
Martin Urban's avatar
Martin Urban committed
677

678
679
    def subscribe(
            self, id, path, pattern=None, reverse=False, hide_hidden=True):
680
681
        if not path:
            return self.unsubscribe(id)
Martin Urban's avatar
Martin Urban committed
682

683
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
Martin Urban's avatar
Martin Urban committed
684

685
686
687
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
Martin Urban's avatar
Martin Urban committed
688

689
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
Martin Urban's avatar
Martin Urban committed
690

691
692
693
694
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
695
696
                    if False not in map(
                            lambda e, c: c is None or e == c, subscriber.id, id):
697
698
699
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
Martin Urban's avatar
Martin Urban committed
700

701
702
    def stop(self):
        self.run = False
Martin Urban's avatar
Martin Urban committed
703

704
705
706
707
708
709
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
Martin Urban's avatar
Martin Urban committed
710
                        if event.action_name in ['delete self', 'move self']:
711
712
713
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
Martin Urban's avatar
Martin Urban committed
714
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
715
716
717
718
719
720
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
721
722
                                    event.watch.mtime = os.path.getmtime(
                                        event.watch.path)
723
724
725
726
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
Martin Urban's avatar
Martin Urban committed
727

728
729
730
731
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
Martin Urban's avatar
Martin Urban committed
732

733
        for subscriber in self.subscribers.items():
734
            subscriber.destroy()
Martin Urban's avatar
Martin Urban committed
735

736
737
        self.monitor.remove_all_watches()
        self.monitor.close()
Martin Urban's avatar
Martin Urban committed
738
739
740


class WatchSubscriber(object):  # this should never be instanced manually
741
    EVENT_DELAYS = {
Martin Urban's avatar
Martin Urban committed
742
743
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
744
    }
745
746
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
Martin Urban's avatar
Martin Urban committed
747

748
749
750
751
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
752
753
754
            raise RuntimeError(
                "There is already a subscriber with this id: " +
                str(id))
755
756
757
758
759
760
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
761
        self.hide_hidden = None
762
        self.event_buffer = {}
763
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
764

765
766
767
768
769
770
771
772
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
773
        del self.subject_buffer
Martin Urban's avatar
Martin Urban committed
774

775
    def process(self, event, subject=""):
776
777
778
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
779
780
            if self.pattern and bool(
                    self.pattern.search(subject)) != self.reverse:
781
                return
Martin Urban's avatar
Martin Urban committed
782

783
784
785
786
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
Martin Urban's avatar
Martin Urban committed
787

788
789
790
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
791
                self.event_buffer[event][1] = now + \
792
                                              WatchSubscriber.EVENT_DELAYS[event][1]
793
            else:
Martin Urban's avatar
Martin Urban committed
794
795
                self.event_buffer[event] = [now + delay for delay in
                                            WatchSubscriber.EVENT_DELAYS[event]]  # first & last event
796
797
798
799
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
Martin Urban's avatar
Martin Urban committed
800

801
802
803
804
805
806
807
808
809
810
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
Martin Urban's avatar
Martin Urban committed
811
        if len(self.id) == 3:  # window_id, view_id, watch_id
812
813
814
815
816
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
817
818
819
820
821
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
Martin Urban's avatar
Martin Urban committed
822
823
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in
                                                 self.subject_buffer[event]]
824
825
826
827
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
828
                data['mtime'] = self.watch.mtime
829
830
831
832
833
            vispa.remote.send_topic(
                "extension.%s.socket.watch" %
                self.id[1],
                window_id=self.id[0],
                data=data)
Martin Urban's avatar
Martin Urban committed
834
        elif len(self.id) == 2:  # userid, workspaceid
835
836
837
838
839
840
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
Martin Urban's avatar
Martin Urban committed
841

842
    def update(self, path, pattern="", reverse=False, hide_hidden=True):
843
844
845
846
847
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
Martin Urban's avatar
Martin Urban committed
848
849
850
            self.subject_buffer = {event: [
                subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
            ] for event, subjects in self.subject_buffer.items()}
851
852
853
        else:
            self.pattern = None
            self.reverse = None
854
        self.hide_hidden = hide_hidden
Martin Urban's avatar
Martin Urban committed
855

856
857
858
859
860
861
    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()
Martin Urban's avatar
Martin Urban committed
862

863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]
Martin Urban's avatar
Martin Urban committed
879

880
881
882
        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)
Martin Urban's avatar
Martin Urban committed
883

884
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
885

886
887
    def unbind(self):
        if not self.watch:
888
            return
Martin Urban's avatar
Martin Urban committed
889

890
891
892
893
        self.watch.subscribers.remove(self)
        if len(self.watch.subscribers) == 0:
            del self.service.watches[self.watch.path]
            self.service.monitor.remove_watch(self.watch)
Martin Urban's avatar
Martin Urban committed
894

895
        self.watch = None
896

Martin Urban's avatar
Martin Urban committed
897

Marcel's avatar
Marcel committed
898
899
900
901
902
903
904
905
def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1

Martin Urban's avatar
Martin Urban committed
906

Marcel's avatar
Marcel committed
907
908
909
910
911
912
913
914
915
def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1