Die Migration der Bereiche "Docker Registry" und "Artifiacts" ist fast abgeschlossen. Die letzten Daten werden im Laufe des heutigen Abend (05.08.2021) noch vollständig hochgeladen. Das Anlegen neuer Images und Artifacts funktioniert bereits wieder.

filesystem.py 35.2 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
import mimetypes
7
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
Gero Müller's avatar
Gero Müller committed
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
from subprocess import call

"""Note: We also monkey-patch subprocess for python 2.6 to
give feature parity with later versions.
"""
try:
    from subprocess import STDOUT, check_output, CalledProcessError
except ImportError:  # pragma: no cover
    # python 2.6 doesn't include check_output
    # monkey patch it in!
    import subprocess
    STDOUT = subprocess.STDOUT

    def check_output(*popenargs, **kwargs):
        if 'stdout' in kwargs:  # pragma: no cover
            raise ValueError('stdout argument not allowed, '
                             'it will be overridden.')
        process = subprocess.Popen(stdout=subprocess.PIPE,
                                   *popenargs, **kwargs)
        output, _ = process.communicate()
        retcode = process.poll()
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
            raise subprocess.CalledProcessError(retcode, cmd,
                                                output=output)
        return output
    subprocess.check_output = check_output

    # overwrite CalledProcessError due to `output`
    # keyword not being available (in 2.6)
    class CalledProcessError(Exception):

        def __init__(self, returncode, cmd, output=None):
            self.returncode = returncode
            self.cmd = cmd
            self.output = output

        def __str__(self):
            return "Command '%s' returned non-zero exit status %d" % (
                self.cmd, self.returncode)
    subprocess.CalledProcessError = CalledProcessError

54
import ConfigParser
55
import json
56
import logging
57
import os, sys
58
59
import re
import shutil
Gero Müller's avatar
Gero Müller committed
60
import stat
61
import subprocess
62
from fsmonitor.polling import FSMonitor
63
import vispa
64
import tempfile
65
66
from pwd import getpwall
from grp import getgrall
67
68
69

try:
    import Image
Gero Müller's avatar
Gero Müller committed
70
    import ImageFilter
Martin Urban's avatar
Martin Urban committed
71

72
73
74
75
76
77
78
79
80
81
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
82

83
84
logger = logging.getLogger(__name__)

Martin Urban's avatar
Martin Urban committed
85

86
87
88
89
90
91
92
93
94
95
96
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
Martin Urban's avatar
Martin Urban committed
97

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

asseldonk's avatar
asseldonk committed
117

Martin Urban's avatar
Martin Urban committed
118
class FileSystem(object):
119
120
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
121
122
123
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
124
        "root": "text/plain"
Marcel's avatar
Marcel committed
125
    }
126
127
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
128

129
    def __init__(self, userid, workspaceid):
130
131
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
132
        self.watchservice = WatchService()
133
134
        self._userid = userid
        self._workspaceid = workspaceid
135
        mimetypes.init()
Martin Urban's avatar
Martin Urban committed
136

137
138
    def __del__(self):
        self.close()
139

140
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
141
142
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
143
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
144
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
145
146
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
147
148
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
149
        else:
Gero Müller's avatar
Gero Müller committed
150
            os.makedirs(self.basedir, 0o700)
151
            return "Basedir now exists"
152

153
    def close(self):
154
155
        if self.watchservice:
            self.watchservice.stop()
Martin Urban's avatar
Martin Urban committed
156

murban's avatar
murban committed
157
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
158
        filepath = self.expand(filepath)
159
        mime, encoding = mimetypes.guess_type(filepath)
160
161
162
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
163
164
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
165
166
167
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
168
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
169
        path = self.expand(path)
170
171
172
173
174
175
176
177
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
178
179
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
180
        path = self.expand(path)
181
182
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
183
184
            return None
        # type correct?
185
186
187
188
189
190
191
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
192

193
194
    def get_file_count(
            self, path, window_id=None, view_id=None, watch_id=None):
195
196
197
198
199
200
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
201
202
        path = self.expand(path)
        if os.path.exists(path):
Martin Urban's avatar
Martin Urban committed
203
204
205
206
207
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
208
209
210
        else:
            return -1

211
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
212
                      filter=None, reverse=False,
213
214
215
216
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
217
218
            if self.watch(
                    path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
219
220
221
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
222
        filter = re.compile(filter) if filter else None
223
        base = self.expand(path)
Martin Urban's avatar
Martin Urban committed
224

225
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
Martin Urban's avatar
Martin Urban committed
226
227
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
228
229
        # ignore failed file info (e.g. access error)
        filelist = [i for i in filelist if 'size' in i]
230
231

        # Determine the parent
232
233
234
235
236
237
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
238
        if encode_json:
Benjamin Fischer's avatar
Benjamin Fischer committed
239
            return json.dumps(data)
240
241
        else:
            return data
Martin Urban's avatar
Martin Urban committed
242

Gero Müller's avatar
Gero Müller committed
243
244
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
245
246
247
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
248
249
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
250
            # dir case
Marcel's avatar
Marcel committed
251
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
252
253
254
255
256
257
258
259
260
261
262
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
263
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
264
265
266
267
268
269
270
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
271
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
272
273
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
274
275
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
276
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
277
278
279
280
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
281
282
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
283
284
285
                suggestion += '/'
            suggestions.append(suggestion)

Benjamin Fischer's avatar
Benjamin Fischer committed
286
        return suggestions if not encode_json else json.dumps(suggestions)
287

murban's avatar
murban committed
288
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
289
        path = self.expand(path)
murban's avatar
murban committed
290
        path = path[1:] if path.startswith(os.sep) else path
291
292
        if path == "":
            return path
murban's avatar
murban committed
293
        path = path[:-1] if path.endswith(os.sep) else path
294
295
        return path

murban's avatar
murban committed
296
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
297
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
298
299
        name = self.expand(name)

Marcel's avatar
Marcel committed
300
        fullpath = os.path.join(path, name)
301
302
303
        try:
            os.mkdir(fullpath)
        except Exception as e:
304
305
            return str(e)
        return ""
306

murban's avatar
murban committed
307
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
308
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
309
310
        name = self.expand(name)

Marcel's avatar
Marcel committed
311
        fullpath = os.path.join(path, name)
312
        try:
313
314
            with file(fullpath, "w") as f:
                pass
315
        except Exception as e:
316
317
            return str(e)
        return ""
318

319
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
320
321
322
323
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

324
325
326
327
328
        if force == False:
            new_name = self.handle_file_name_collision(new_name, path)
        name = os.path.join(path, name)
        new_name = os.path.join(path, new_name)
        os.renames(name, new_name)
329

murban's avatar
murban committed
330
331
    def remove(self, path):
        if isinstance(path, list):
332
333
334
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
335
            path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
336
337
338
            if os.path.islink(path):
                os.unlink(path)
            elif os.path.isdir(path):
Marcel's avatar
Marcel committed
339
340
341
                shutil.rmtree(path)
            else:
                os.remove(path)
342

343
    def move(self, source, destination):
344
345
346
347
348
349
        if isinstance(source, list):
            for s in source:
                self.move(s, destination)
        else:
            source = self.expand(source)
            destination = self.expand(destination)
350
351
352
353
            name = os.path.split(source)[1]
            newname = self.handle_file_name_collision(name, destination)
            destination = os.path.join(destination, newname)
            if os.path.isdir(source):
354
                shutil.copytree(source, destination, symlinks=True)
355
356
357
358
359
                shutil.rmtree(source)
            else:
                shutil.copy2(source, destination)
                os.remove(source)

360

361
    def compress(self, paths, path, name, is_tmp=False):
Gero Müller's avatar
Gero Müller committed
362
363
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
364
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
365

366
        if is_tmp:
367
368
369
370
371
            tempdir = tempfile._get_default_tempdir()
            name = name or next(tempfile._get_candidate_names())
            name = name if name.endswith(".zip") else name + ".zip"
            name = self.handle_file_name_collision(name, tempdir)
            fullpath = os.path.join(tempdir, name)
372
373
374
375
376
377
        else:
            path = path if not path.endswith(os.sep) else path[:-1]
            path = self.expand(path)
            name = name if name.endswith(".zip") else name + ".zip"
            name = self.handle_file_name_collision(name, path)
            fullpath = os.path.join(path, name)
Gero Müller's avatar
Gero Müller committed
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
394
                            ap = fullp[
395
                                 len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
396
397
398
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
399
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
400
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
401
                    archive.write(p, ap)
402

403
404
        return fullpath

405
    def decompress(self, path):
406
        # filepath and extract path
407
408
409
410
        path = self.expand(path)

        # "foo/bar/file.zip" -> ("foo/bar", "file")
        dst = os.path.split(os.path.splitext(path)[0])
411

412
413
414
        with ZipFile(path, "r") as archive:
            dstdir = os.path.join(dst[0], self.handle_file_name_collision(dst[1], dst[0]))
            archive.extractall(dstdir)
415

416
417
418
    def paste(self, path, fullsrc, cut):
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
murban's avatar
murban committed
419
                self.paste(path, p, cut)
420
421
            return True

422
423
424
425
426
        path = self.expand(path)
        fullsrc = self.expand(fullsrc)
        srcdir, srcname = os.path.split(fullsrc)

        target = self.handle_file_name_collision(srcname, path)
427
        fulltarget = os.path.join(path, target)
428

429
430
        if cut and self.checkPermissions(srcdir):
            shutil.move(fullsrc, fulltarget)
431
        else:
432
433
434
435
            if os.path.isdir(fullsrc):
                shutil.copytree(fullsrc, fulltarget, symlinks=True)
            else:
                shutil.copy2(fullsrc, fulltarget)
436

437
    def save_file(self, path, content, force=True, binary=False,
438
                  utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
439
        path = self.expand(path)
Marcel's avatar
Marcel committed
440
        # check if file already exists
murban's avatar
murban committed
441
        if os.path.exists(path) and not force:
442
443
444
445
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
Benjamin Fischer's avatar
Benjamin Fischer committed
446
            })
447
448
449
450
451
452
453
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
454
455
            mtime = 0

456
457
458
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
459
        else:
460
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
461

462
463
        return json.dumps({
            "mtime": os.path.getmtime(path),
464
465
            # save is not successful, if file not writable
            "success": mtime > 0 and self.checkPermissions(path),
Martin Urban's avatar
Martin Urban committed
466
            "watch_error": watch_error,
467
            "path": path
Benjamin Fischer's avatar
Benjamin Fischer committed
468
        })
Martin Urban's avatar
Martin Urban committed
469

470
    def get_file(self, path, binary=False,
471
                 utf8=False, window_id=None, view_id=None, watch_id=None, max_size=20):
472
473
474
475
476
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
477
        max_size *= 1024*1024#
Martin Urban's avatar
Martin Urban committed
478

479
480
        # actual function
        path = self.expand(path)
481

482
        try:
483
484
485
486
487
488
489
490
491
            if os.path.getsize(path) > max_size:
                content = ""

                with open(path, "rb" if binary else "r") as f:
                    while sys.getsizeof(content) < max_size:
                        content += f.next()
                writable = False
                error = None
                size_limit = True
492

493
494
            else:
                with open(path, "rb" if binary else "r") as f:
495
                    content = f.read()
496
497
498
499

                writable = self.checkPermissions(path)
                error = None
                size_limit = False
500

501
502
            if utf8:
                content = content.decode('utf8')
503

504
505
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
506
            mtime = 0
507
            content = ""
508
            size_limit = False
509
            writable = None
510
            error = str(e)
Martin Urban's avatar
Martin Urban committed
511

512
513
514
515
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
516
            "watch_error": watch_error,
Martin Urban's avatar
Martin Urban committed
517
            "writable": writable,
518
519
            "size_limit": size_limit,
            "max_size": max_size/(1024*1024),
520
            "error": error
Benjamin Fischer's avatar
Benjamin Fischer committed
521
        })
522

Martin Urban's avatar
Martin Urban committed
523
524
    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)
525

526
    def getfacl(self, path):
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
        path = self.expand(path)
        s = os.stat(path)
        ret = dict(stat=dict((k[3:], getattr(s, k)) for k in dir(s) if k.startswith("st_")))
        try:
            ret["facl"] = list(check_output(["getfacl", "-pc", path]).splitlines())
        except CalledProcessError:
            pass
        try:
            ret["nfs4_facl"] = list(check_output(["nfs4_getfacl", path]).splitlines())
        except CalledProcessError:
            pass
        return ret

    def get_user_and_group_ids(self):
        return dict(
            users=dict((s.pw_name, s.pw_uid) for s in getpwall()),
            groups=dict((s.gr_name, s.gr_gid) for s in getgrall()),
        )
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570

    def setfacl(self, path, type, name, mode, remove=False, recursive=False, default=False):
        if type not in ("user", "group", "mask", "other"):
            raise TypeError("type '%s' not in ('user', 'group', 'mask', 'other')" % type)
        if type in ("mask", "other"):
            name = ""

        arguments = ["setfacl"]

        if default:
            arguments.append("-d")

        if recursive:
            arguments.append("-R")

        if remove:
            action = "-x"
            acl = "%s:%s" % (type, name)
        else:
            action = "-m"
            acl = "%s:%s:%s" % (type, name, mode)
        arguments.append(action)
        arguments.append(acl)
        arguments.append(path)
        call(arguments, stderr=open(os.devnull, "wb"))

Gero Müller's avatar
Gero Müller committed
571
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
572
                          path=None, force=True, append=False):
573
574
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Martin Urban's avatar
Martin Urban committed
575
        # return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
576
577
578
579
580

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

581
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
582
583
584
585
586
587
588
589
590
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

591
592
593
594
    def get_file_content(self, path, offset=0, length=None):
        with open(self.expand(path), "rb") as f:
            f.seek(offset)
            content = f.read() if length is None else f.read(length)
Marcel's avatar
Marcel committed
595
        return content
596

Marcel's avatar
Marcel committed
597
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
598
        path = self.expand(path)
Marcel's avatar
Marcel committed
599
600
        return os.path.getmtime(path)

601
602
603
604
    def stat(self, path):
        path = self.expand(path)
        return os.stat(path)

murban's avatar
murban committed
605
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
606
        path = self.expand(path)
607
        extension = path.split(".")[-1]
608
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
609

murban's avatar
murban committed
610
    def handle_file_name_collision(self, name, path):
611
        # collision?
612
        path = self.expand(path)
murban's avatar
murban committed
613
        files = os.listdir(path)
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
631
        counter = prename.split("_copy")[-1]
632
633
634
635
636

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
637
                preprename = "_copy".join(prename.split("_copy")[:-1])
638
639
640
641
642
643
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
644
            newname = "%s_copy%d%s" % (preprename,
Gero Müller's avatar
Gero Müller committed
645
646
                                   counter,
                                   "" if extension == "" else "." + extension)
647
        else:
648
            newname = "%s_copy1%s" % (
Gero Müller's avatar
Gero Müller committed
649
                prename, "" if extension == "" else "." + extension)
650
651

        # return
murban's avatar
murban committed
652
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
653

Marcel's avatar
Marcel committed
654
655
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
656

Gero Müller's avatar
Gero Müller committed
657
    def thumbnail(self, path, width=100, height=100, sharpen=True):
658
659
660
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
661
662
663
664
            try:
                img = Image.open(path).convert('RGB')
            except IOError:
                return None
Gero Müller's avatar
Gero Müller committed
665
            img.thumbnail((width, height), Image.ANTIALIAS)
666
            img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
667
            img.save(output, "JPEG")
668
669
670
671
672
673
674
675
676
677
678
679
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

680
681
    def watch(self, path, window_id, view_id, watch_id,
              pattern=None, reverse=False, hide_hidden=True):
682
683
684
685
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
Martin Urban's avatar
Martin Urban committed
686
687
688
689
690

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

691
692
        #first: remove the old watch
        self.unwatch(window_id, view_id, watch_id)
693

694
695
696
697
698
699
700
701
        self.watchservice.subscribe(
            (window_id,
             view_id,
             watch_id),
            path,
            pattern,
            reverse,
            hide_hidden)
702
        return ""
Martin Urban's avatar
Martin Urban committed
703

704
    def unwatch(self, window_id, view_id, watch_id=None):
705
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
706
707
        return ""

708
    def get_workspaceini(self, request, fail_on_missing=False):
709
        try:
710
            request_dict = json.loads(request)
711
            config = ConfigParser.ConfigParser()
712
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
Martin Urban's avatar
Martin Urban committed
713
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
714
715
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
Martin Urban's avatar
Martin Urban committed
716
                self._watch_workspaceini()
717
718
            else:
                mtime = -1
719
            if not isinstance(request_dict, dict):
720
                request_dict = dict.fromkeys(config.sections(), True)
721
722
723
724
725
726
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
727
                        data[section] = dict(config.items(section))
728
729
730
731
732
733
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
Martin Urban's avatar
Martin Urban committed
734
735
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
736
                elif fail_on_missing:
737
738
739
                    raise Exception(
                        'workspace.ini is missing the section [%s]' %
                        section)
740
            return json.dumps({
741
                "content": data,
742
743
                "success": True,
                "mtime": mtime
Benjamin Fischer's avatar
Benjamin Fischer committed
744
            })
745
746
        except Exception as e:
            return json.dumps({
747
748
749
                "content": "",
                "success": False,
                "error": str(e)
Benjamin Fischer's avatar
Benjamin Fischer committed
750
            })
751

752
    def set_workspaceini(self, request):
753
        try:
754
755
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
756
757
                raise Exception(
                    'Given values to be set in workspace.ini in wrong format')
758
            filename = self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)
759
            config = ConfigParser.ConfigParser()
760
            config.read(filename)
761
762
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
763
764
                    raise Exception(
                        'Given values to be set in workspace.ini in wrong format')
765
766
767
768
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
769
770
771
772
            filedir = os.path.dirname(filename)
            if not os.path.isdir(filedir):
                os.makedirs(filedir)
            with open(filename, 'w') as f:
773
                config.write(f)
774
            self._watch_workspaceini()
775
            return ""
776
        except Exception as e:
777
            return str(e)
asseldonk's avatar
asseldonk committed
778

779
    def _watch_workspaceini(self):
780
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
781
782
783
784
            self.watchservice.subscribe(
                (self._userid,
                 self._workspaceid),
                FileSystem.PRIVATE_WORKSPACE_CONF)
785

Martin Urban's avatar
Martin Urban committed
786

787
788
789
790
791
792
class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
793
        self.monitor = FSMonitor()
794
795
        self.run = True
        self.thread = Thread(target=self._worker)
796
        self.thread.daemon = True
797
        self.thread.start()
Martin Urban's avatar
Martin Urban committed
798

799
800
    def subscribe(
            self, id, path, pattern=None, reverse=False, hide_hidden=True):
801
802
        if not path:
            return self.unsubscribe(id)
Martin Urban's avatar
Martin Urban committed
803

804
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
Martin Urban's avatar
Martin Urban committed
805

806
807
808
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
Martin Urban's avatar
Martin Urban committed
809

810
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
Martin Urban's avatar
Martin Urban committed
811

812
813
814
815
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
816
817
                    if False not in map(
                            lambda e, c: c is None or e == c, subscriber.id, id):
818
819
820
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
Martin Urban's avatar
Martin Urban committed
821

822
823
    def stop(self):
        self.run = False
Martin Urban's avatar
Martin Urban committed
824

825
826
827
828
829
830
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
Martin Urban's avatar
Martin Urban committed
831
                        if event.action_name in ['delete self', 'move self']:
832
833
834
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
Martin Urban's avatar
Martin Urban committed
835
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
836
837
838
839
840
841
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
842
843
                                    event.watch.mtime = os.path.getmtime(
                                        event.watch.path)
844
845
846
847
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
Martin Urban's avatar
Martin Urban committed
848

849
850
851
852
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
Martin Urban's avatar
Martin Urban committed
853

Gero Müller's avatar
Gero Müller committed
854
        for subscriber in self.subscribers.values():
855
            subscriber.destroy()
Martin Urban's avatar
Martin Urban committed
856

857
        self.monitor.remove_all_watches()
858
        del self.monitor
Martin Urban's avatar
Martin Urban committed
859
860
861


class WatchSubscriber(object):  # this should never be instanced manually
862
    EVENT_DELAYS = {
Martin Urban's avatar
Martin Urban committed
863
864
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
865
    }
866
867
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
Martin Urban's avatar
Martin Urban committed
868

869
870
871
872
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
873
874
875
            raise RuntimeError(
                "There is already a subscriber with this id: " +
                str(id))
876
877
878
879
880
881
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
882
        self.hide_hidden = None
883
        self.event_buffer = {}
884
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
885

886
887
888
889
890
891
892
893
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
894
        del self.subject_buffer
Martin Urban's avatar
Martin Urban committed
895

896
    def process(self, event, subject=""):
897
898
899
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
900
901
            if self.pattern and bool(
                    self.pattern.search(subject)) != self.reverse:
902
                return
Martin Urban's avatar
Martin Urban committed
903

904
905
906
907
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
Martin Urban's avatar
Martin Urban committed
908

909
910
911
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer: