filesystem.py 35.1 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
import mimetypes
7
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
Gero Müller's avatar
Gero Müller committed
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
from subprocess import call

"""Note: We also monkey-patch subprocess for python 2.6 to
give feature parity with later versions.
"""
try:
    from subprocess import STDOUT, check_output, CalledProcessError
except ImportError:  # pragma: no cover
    # python 2.6 doesn't include check_output
    # monkey patch it in!
    import subprocess
    STDOUT = subprocess.STDOUT

    def check_output(*popenargs, **kwargs):
        if 'stdout' in kwargs:  # pragma: no cover
            raise ValueError('stdout argument not allowed, '
                             'it will be overridden.')
        process = subprocess.Popen(stdout=subprocess.PIPE,
                                   *popenargs, **kwargs)
        output, _ = process.communicate()
        retcode = process.poll()
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
            raise subprocess.CalledProcessError(retcode, cmd,
                                                output=output)
        return output
    subprocess.check_output = check_output

    # overwrite CalledProcessError due to `output`
    # keyword not being available (in 2.6)
    class CalledProcessError(Exception):

        def __init__(self, returncode, cmd, output=None):
            self.returncode = returncode
            self.cmd = cmd
            self.output = output

        def __str__(self):
            return "Command '%s' returned non-zero exit status %d" % (
                self.cmd, self.returncode)
    subprocess.CalledProcessError = CalledProcessError

54
import ConfigParser
55
import json
56
import logging
57
import os, sys
58
59
import re
import shutil
Gero Müller's avatar
Gero Müller committed
60
import stat
61
import subprocess
62
63
import fsmonitor
import vispa
64
import tempfile
65
66
67

try:
    import Image
Gero Müller's avatar
Gero Müller committed
68
    import ImageFilter
Martin Urban's avatar
Martin Urban committed
69

70
71
72
73
74
75
76
77
78
79
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
80

81
82
logger = logging.getLogger(__name__)

Martin Urban's avatar
Martin Urban committed
83

84
85
86
87
88
89
90
91
92
93
94
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
Martin Urban's avatar
Martin Urban committed
95

96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

asseldonk's avatar
asseldonk committed
115

Martin Urban's avatar
Martin Urban committed
116
class FileSystem(object):
117
118
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
119
120
121
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
122
        "root": "text/plain"
Marcel's avatar
Marcel committed
123
    }
124
125
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
126

127
    def __init__(self, userid, workspaceid):
128
129
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
130
        self.watchservice = WatchService()
131
132
        self._userid = userid
        self._workspaceid = workspaceid
133
        mimetypes.init()
Martin Urban's avatar
Martin Urban committed
134

135
136
    def __del__(self):
        self.close()
137

138
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
139
140
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
141
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
142
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
143
144
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
145
146
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
147
        else:
Gero Müller's avatar
Gero Müller committed
148
            os.makedirs(self.basedir, 0o700)
149
            return "Basedir now exists"
150

151
    def close(self):
152
153
        if self.watchservice:
            self.watchservice.stop()
Martin Urban's avatar
Martin Urban committed
154

murban's avatar
murban committed
155
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
156
        filepath = self.expand(filepath)
157
        mime, encoding = mimetypes.guess_type(filepath)
158
159
160
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
161
162
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
163
164
165
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
166
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
167
        path = self.expand(path)
168
169
170
171
172
173
174
175
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
176
177
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
178
        path = self.expand(path)
179
180
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
181
182
            return None
        # type correct?
183
184
185
186
187
188
189
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
190

191
192
    def get_file_count(
            self, path, window_id=None, view_id=None, watch_id=None):
193
194
195
196
197
198
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
199
200
        path = self.expand(path)
        if os.path.exists(path):
Martin Urban's avatar
Martin Urban committed
201
202
203
204
205
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
206
207
208
        else:
            return -1

209
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
210
                      filter=None, reverse=False,
211
212
213
214
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
215
216
            if self.watch(
                    path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
217
218
219
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
220
        filter = re.compile(filter) if filter else None
221
        base = self.expand(path)
Martin Urban's avatar
Martin Urban committed
222

223
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
Martin Urban's avatar
Martin Urban committed
224
225
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
226
227
        # ignore failed file info (e.g. access error)
        filelist = [i for i in filelist if 'size' in i]
228
229

        # Determine the parent
230
231
232
233
234
235
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
236
237
        if encode_json:
            return json.dumps(data)
238
239
        else:
            return data
Martin Urban's avatar
Martin Urban committed
240

Gero Müller's avatar
Gero Müller committed
241
242
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
243
244
245
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
246
247
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
248
            # dir case
Marcel's avatar
Marcel committed
249
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
250
251
252
253
254
255
256
257
258
259
260
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
261
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
262
263
264
265
266
267
268
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
269
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
270
271
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
272
273
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
274
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
275
276
277
278
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
279
280
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
281
282
283
                suggestion += '/'
            suggestions.append(suggestion)

284
        return suggestions if not encode_json else json.dumps(suggestions)
285

murban's avatar
murban committed
286
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
287
        path = self.expand(path)
murban's avatar
murban committed
288
        path = path[1:] if path.startswith(os.sep) else path
289
290
        if path == "":
            return path
murban's avatar
murban committed
291
        path = path[:-1] if path.endswith(os.sep) else path
292
293
        return path

murban's avatar
murban committed
294
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
295
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
296
297
        name = self.expand(name)

298
        # folder with the same name existent?
299
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
300
        fullpath = os.path.join(path, name)
301
302
303
        try:
            os.mkdir(fullpath)
        except Exception as e:
304
            # raise Exception("You don't have the permission to create this folder!")
305
306
            raise Exception(str(e))

murban's avatar
murban committed
307
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
308
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
309
310
        name = self.expand(name)

311
        # file with the same name existent?
312
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
313
        fullpath = os.path.join(path, name)
314
315
316
317
318
319
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

320
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
321
322
323
324
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

325
326
327
328
329
        if force == False:
            new_name = self.handle_file_name_collision(new_name, path)
        name = os.path.join(path, name)
        new_name = os.path.join(path, new_name)
        os.renames(name, new_name)
330

murban's avatar
murban committed
331
332
    def remove(self, path):
        if isinstance(path, list):
333
334
335
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
336
            path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
337
338
339
            if os.path.islink(path):
                os.unlink(path)
            elif os.path.isdir(path):
Marcel's avatar
Marcel committed
340
341
342
                shutil.rmtree(path)
            else:
                os.remove(path)
343

344
    def move(self, source, destination):
345
346
347
348
349
350
        if isinstance(source, list):
            for s in source:
                self.move(s, destination)
        else:
            source = self.expand(source)
            destination = self.expand(destination)
351
352
353
354
            name = os.path.split(source)[1]
            newname = self.handle_file_name_collision(name, destination)
            destination = os.path.join(destination, newname)
            if os.path.isdir(source):
355
                shutil.copytree(source, destination, symlinks=True)
356
357
358
359
360
                shutil.rmtree(source)
            else:
                shutil.copy2(source, destination)
                os.remove(source)

361

362
    def compress(self, paths, path, name, is_tmp=False):
Gero Müller's avatar
Gero Müller committed
363
364
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
365
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
366

367
        if is_tmp:
368
369
370
371
372
            tempdir = tempfile._get_default_tempdir()
            name = name or next(tempfile._get_candidate_names())
            name = name if name.endswith(".zip") else name + ".zip"
            name = self.handle_file_name_collision(name, tempdir)
            fullpath = os.path.join(tempdir, name)
373
374
375
376
377
378
        else:
            path = path if not path.endswith(os.sep) else path[:-1]
            path = self.expand(path)
            name = name if name.endswith(".zip") else name + ".zip"
            name = self.handle_file_name_collision(name, path)
            fullpath = os.path.join(path, name)
Gero Müller's avatar
Gero Müller committed
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
395
                            ap = fullp[
396
                                 len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
397
398
399
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
400
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
401
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
402
                    archive.write(p, ap)
403

404
405
        return fullpath

406
    def decompress(self, path):
407
        # filepath and extract path
408
409
410
411
        path = self.expand(path)

        # "foo/bar/file.zip" -> ("foo/bar", "file")
        dst = os.path.split(os.path.splitext(path)[0])
412

413
414
415
        with ZipFile(path, "r") as archive:
            dstdir = os.path.join(dst[0], self.handle_file_name_collision(dst[1], dst[0]))
            archive.extractall(dstdir)
416

417
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
418
        # TODO
419
420
421
422
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
423
                self.paste(path, p, cut)
424
425
            return True

426
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
Gero Müller's avatar
Gero Müller committed
427
428
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
429
        fulltarget = os.path.join(path, target)
430

asseldonk's avatar
asseldonk committed
431
        if os.path.isdir(fullsrc):
432
            shutil.copytree(fullsrc, fulltarget, symlinks=True)
asseldonk's avatar
asseldonk committed
433
434
            if cut:
                shutil.rmtree(fullsrc)
435
        else:
asseldonk's avatar
asseldonk committed
436
437
438
            shutil.copy2(fullsrc, fulltarget)
            if cut:
                os.remove(fullsrc)
439

440
    def save_file(self, path, content, force=True, binary=False,
441
                  utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
442
        path = self.expand(path)
Marcel's avatar
Marcel committed
443
        # check if file already exists
murban's avatar
murban committed
444
        if os.path.exists(path) and not force:
445
446
447
448
449
450
451
452
453
454
455
456
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
457
458
            mtime = 0

459
460
461
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
462
        else:
463
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
464

465
466
        return json.dumps({
            "mtime": os.path.getmtime(path),
467
468
            # save is not successful, if file not writable
            "success": mtime > 0 and self.checkPermissions(path),
Martin Urban's avatar
Martin Urban committed
469
            "watch_error": watch_error,
470
            "path": path
471
        })
Martin Urban's avatar
Martin Urban committed
472

473
    def get_file(self, path, binary=False,
474
                 utf8=False, window_id=None, view_id=None, watch_id=None, max_size=20):
475
476
477
478
479
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
480
        max_size *= 1024*1024#
Martin Urban's avatar
Martin Urban committed
481

482
483
        # actual function
        path = self.expand(path)
484
        
485
        try:
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
            if os.path.getsize(path) > max_size:
                content = ""

                with open(path, "rb" if binary else "r") as f:
                    while sys.getsizeof(content) < max_size:
                        content += f.next()
                writable = False
                error = None
                size_limit = True
            
            else:
                with open(path, "rb" if binary else "r") as f:
                    content = f.read()   

                writable = self.checkPermissions(path)
                error = None
                size_limit = False
            
504
505
            if utf8:
                content = content.decode('utf8')
506

507
508
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
509
            mtime = 0
510
            content = ""
511
            size_limit = False
512
            writable = None
513
            error = str(e)
Martin Urban's avatar
Martin Urban committed
514

515
516
517
518
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
519
            "watch_error": watch_error,
Martin Urban's avatar
Martin Urban committed
520
            "writable": writable,
521
522
            "size_limit": size_limit,
            "max_size": max_size/(1024*1024),
523
            "error": error
524
        })
525

Martin Urban's avatar
Martin Urban committed
526
527
    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)
528

529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
    def getfacl(self, path):
        o = check_output(["getfacl", "-p", "-t", path])
        facls = []
        for line in o.splitlines():
            if not line: continue
            if line.startswith("#"): continue
            e = line.split()
            if e[0] in ["mask", "other"]: e[1:1] = [""]
            facls.append(tuple(e))
        return facls

    def setfacl(self, path, type, name, mode, remove=False, recursive=False, default=False):
        if type not in ("user", "group", "mask", "other"):
            raise TypeError("type '%s' not in ('user', 'group', 'mask', 'other')" % type)
        if type in ("mask", "other"):
            name = ""

        arguments = ["setfacl"]

        if default:
            arguments.append("-d")

        if recursive:
            arguments.append("-R")

        if remove:
            action = "-x"
            acl = "%s:%s" % (type, name)
        else:
            action = "-m"
            acl = "%s:%s:%s" % (type, name, mode)
        arguments.append(action)
        arguments.append(acl)
        arguments.append(path)
        call(arguments, stderr=open(os.devnull, "wb"))

Gero Müller's avatar
Gero Müller committed
565
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
566
                          path=None, force=True, append=False):
567
568
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Martin Urban's avatar
Martin Urban committed
569
        # return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
570
571
572
573
574

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

575
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
576
577
578
579
580
581
582
583
584
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

585
586
587
588
    def get_file_content(self, path, offset=0, length=None):
        with open(self.expand(path), "rb") as f:
            f.seek(offset)
            content = f.read() if length is None else f.read(length)
Marcel's avatar
Marcel committed
589
        return content
590

Marcel's avatar
Marcel committed
591
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
592
        path = self.expand(path)
Marcel's avatar
Marcel committed
593
594
        return os.path.getmtime(path)

595
596
597
598
    def stat(self, path):
        path = self.expand(path)
        return os.stat(path)

murban's avatar
murban committed
599
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
600
        path = self.expand(path)
601
        extension = path.split(".")[-1]
602
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
603

murban's avatar
murban committed
604
    def handle_file_name_collision(self, name, path):
605
        # collision?
606
        path = self.expand(path)
murban's avatar
murban committed
607
        files = os.listdir(path)
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
625
        counter = prename.split("_copy")[-1]
626
627
628
629
630

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
631
                preprename = "_copy".join(prename.split("_copy")[:-1])
632
633
634
635
636
637
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
638
            newname = "%s_copy%d%s" % (preprename,
Gero Müller's avatar
Gero Müller committed
639
640
                                   counter,
                                   "" if extension == "" else "." + extension)
641
        else:
642
            newname = "%s_copy1%s" % (
Gero Müller's avatar
Gero Müller committed
643
                prename, "" if extension == "" else "." + extension)
644
645

        # return
murban's avatar
murban committed
646
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
647

Marcel's avatar
Marcel committed
648
649
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
650

Gero Müller's avatar
Gero Müller committed
651
    def thumbnail(self, path, width=100, height=100, sharpen=True):
652
653
654
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
655
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
656
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
657
            if sharpen:
658
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
659
            img.save(output, "JPEG")
660
661
662
663
664
665
666
667
668
669
670
671
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

672
673
    def watch(self, path, window_id, view_id, watch_id,
              pattern=None, reverse=False, hide_hidden=True):
674
675
676
677
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
Martin Urban's avatar
Martin Urban committed
678
679
680
681
682

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

683
684
        #first: remove the old watch
        self.unwatch(window_id, view_id, watch_id)
685

686
687
688
689
690
691
692
693
        self.watchservice.subscribe(
            (window_id,
             view_id,
             watch_id),
            path,
            pattern,
            reverse,
            hide_hidden)
694
        return ""
Martin Urban's avatar
Martin Urban committed
695

696
    def unwatch(self, window_id, view_id, watch_id=None):
697
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
698
699
        return ""

700
    def get_workspaceini(self, request, fail_on_missing=False):
701
        try:
702
            request_dict = json.loads(request)
703
            config = ConfigParser.ConfigParser()
704
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
Martin Urban's avatar
Martin Urban committed
705
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
706
707
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
Martin Urban's avatar
Martin Urban committed
708
                self._watch_workspaceini()
709
710
            else:
                mtime = -1
711
            if not isinstance(request_dict, dict):
712
                request_dict = dict.fromkeys(config.sections(), True)
713
714
715
716
717
718
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
719
                        data[section] = dict(config.items(section))
720
721
722
723
724
725
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
Martin Urban's avatar
Martin Urban committed
726
727
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
728
                elif fail_on_missing:
729
730
731
                    raise Exception(
                        'workspace.ini is missing the section [%s]' %
                        section)
732
            return json.dumps({
733
                "content": data,
734
735
                "success": True,
                "mtime": mtime
Martin Urban's avatar
Martin Urban committed
736
            })
737
738
        except Exception as e:
            return json.dumps({
739
740
741
                "content": "",
                "success": False,
                "error": str(e)
Martin Urban's avatar
Martin Urban committed
742
            })
743

744
    def set_workspaceini(self, request):
745
        try:
746
747
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
748
749
                raise Exception(
                    'Given values to be set in workspace.ini in wrong format')
750
            filename = self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)
751
            config = ConfigParser.ConfigParser()
752
            config.read(filename)
753
754
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
755
756
                    raise Exception(
                        'Given values to be set in workspace.ini in wrong format')
757
758
759
760
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
761
762
763
764
            filedir = os.path.dirname(filename)
            if not os.path.isdir(filedir):
                os.makedirs(filedir)
            with open(filename, 'w') as f:
765
                config.write(f)
766
            self._watch_workspaceini()
767
            return ""
768
        except Exception as e:
769
            return str(e)
asseldonk's avatar
asseldonk committed
770

771
    def _watch_workspaceini(self):
772
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
773
774
775
776
            self.watchservice.subscribe(
                (self._userid,
                 self._workspaceid),
                FileSystem.PRIVATE_WORKSPACE_CONF)
777

Martin Urban's avatar
Martin Urban committed
778

779
780
781
782
783
784
785
786
787
class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
788
        self.thread.daemon = True
789
        self.thread.start()
Martin Urban's avatar
Martin Urban committed
790

791
792
    def subscribe(
            self, id, path, pattern=None, reverse=False, hide_hidden=True):
793
794
        if not path:
            return self.unsubscribe(id)
Martin Urban's avatar
Martin Urban committed
795

796
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
Martin Urban's avatar
Martin Urban committed
797

798
799
800
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
Martin Urban's avatar
Martin Urban committed
801

802
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
Martin Urban's avatar
Martin Urban committed
803

804
805
806
807
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
808
809
                    if False not in map(
                            lambda e, c: c is None or e == c, subscriber.id, id):
810
811
812
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
Martin Urban's avatar
Martin Urban committed
813

814
815
    def stop(self):
        self.run = False
Martin Urban's avatar
Martin Urban committed
816

817
818
819
820
821
822
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
Martin Urban's avatar
Martin Urban committed
823
                        if event.action_name in ['delete self', 'move self']:
824
825
826
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
Martin Urban's avatar
Martin Urban committed
827
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
828
829
830
831
832
833
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
834
835
                                    event.watch.mtime = os.path.getmtime(
                                        event.watch.path)
836
837
838
839
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
Martin Urban's avatar
Martin Urban committed
840

841
842
843
844
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
Martin Urban's avatar
Martin Urban committed
845

Gero Müller's avatar
Gero Müller committed
846
        for subscriber in self.subscribers.values():
847
            subscriber.destroy()
Martin Urban's avatar
Martin Urban committed
848

849
        self.monitor.remove_all_watches()
850
        del self.monitor
Martin Urban's avatar
Martin Urban committed
851
852
853


class WatchSubscriber(object):  # this should never be instanced manually
854
    EVENT_DELAYS = {
Martin Urban's avatar
Martin Urban committed
855
856
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
857
    }
858
859
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
Martin Urban's avatar
Martin Urban committed
860

861
862
863
864
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
865
866
867
            raise RuntimeError(
                "There is already a subscriber with this id: " +
                str(id))
868
869
870
871
872
873
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
874
        self.hide_hidden = None
875
        self.event_buffer = {}
876
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
877

878
879
880
881
882
883
884
885
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
886
        del self.subject_buffer
Martin Urban's avatar
Martin Urban committed
887

888
    def process(self, event, subject=""):
889
890
891
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
892
893
            if self.pattern and bool(
                    self.pattern.search(subject)) != self.reverse:
894
                return
Martin Urban's avatar
Martin Urban committed
895

896
897
898
899
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
Martin Urban's avatar
Martin Urban committed
900

901
902
903
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
904
                self.event_buffer[event][1] = now + \
905
                                              WatchSubscriber.EVENT_DELAYS[event][1]
906
            else:
Martin Urban's avatar
Martin Urban committed
907
908
                self.event_buffer[event] = [now + delay for delay in
                                            WatchSubscriber.EVENT_DELAYS[event]]  # first & last event
909
910
911
912
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
Martin Urban's avatar
Martin Urban committed
913

914
915
916
917
918
919
920
921
922
923
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
Martin Urban's avatar
Martin Urban committed
924
        if len(self.id) == 3:  # window_id, view_id, watch_id
925
926
927
928
929
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
930
931
932
933
934
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
Martin Urban's avatar
Martin Urban committed
935
936
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in
                                                 self.subject_buffer[event]]
937
938
939
940
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
941
                data['mtime'] = self.watch.mtime
942
943
944
945
946
            vispa.remote.send_topic(
                "extension.%s.socket.watch" %
                self.id[1],
                window_id=self.id[0],
                data=data)
Martin Urban's avatar
Martin Urban committed
947
        elif len(self.id) == 2:  # userid, workspaceid
948
949
950
951
952
953
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
Martin Urban's avatar
Martin Urban committed
954

955
    def update(self, path, pattern="", reverse=False, hide_hidden=True):
956
957
958
959
960
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
961
962
963
964
965
966
967
968
            old_subject_buffer = self.subject_buffer
            self.subject_buffer = {}
            for event, subjects in old_subject_buffer.items():
                new_subject_list = [
                    subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
                ]
                if len(new_subject_list):
                    self.