filesystem.py 20.8 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
7
from mimetypes import guess_type
from zipfile import ZipFile
8
from threading import Lock
9
import json
10
11
12
13
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
14
import stat
15
import subprocess
16
17
import fsmonitor
import vispa
18
19
20

try:
    import Image
Gero Müller's avatar
Gero Müller committed
21
    import ImageFilter
22
23
24
25
26
27
28
29
30
31
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
32

33
34
logger = logging.getLogger(__name__)

35

36
37
class FileSystem(object):

38
39
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
40
41
42
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
43
        "root": "text/plain"
Marcel's avatar
Marcel committed
44
    }
45
46
47
48

    def __init__(self):
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
49
50
51
52
53
54
55
        self._monitor_thread = fsmonitor.FSMonitorThread(self._monitor_callback)
        self._monitor_watches = {}
        self._monitor_listener = {}
        self._monitor_lock = Lock()
    
    def __del__(self):
        self.close()
56

57
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
58
59
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
60
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
61
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
62
63
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
64
65
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
66
        else:
Gero Müller's avatar
Gero Müller committed
67
            os.makedirs(self.basedir, 0o700)
68
            return "Basedir now exists"
69

70
71
72
73
74
    def close(self):
        if self._monitor_thread:
            self._monitor_thread.remove_all_watches()
            self._monitor_thread.running = False
    
murban's avatar
murban committed
75
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
76
        filepath = self.expand(filepath)
77
78
79
80
        mime, encoding = guess_type(filepath)
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
81
82
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
83
84
85
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
86
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
87
        path = self.expand(path)
88
89
90
91
92
93
94
95
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
96
97
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
98
        path = self.expand(path)
99
100
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
101
102
            return None
        # type correct?
103
104
105
106
107
108
109
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
110

111
112
113
114
115
116
117
    def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
118
119
120
121
122
123
        path = self.expand(path)
        if os.path.exists(path):
            return len(os.listdir(path))
        else:
            return -1

Gero Müller's avatar
Gero Müller committed
124
125
    def get_file_list(self, path, deep=False,
                      filter=None, reverse=False,
126
127
128
129
130
131
132
133
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
134
        filter = re.compile(filter) if filter else None
135
        filelist = []
Marcel's avatar
Marcel committed
136
        path_expand = self.expand(path)
137
138
139
140
141
142
143
        try:
            for elem in os.listdir(path_expand):
                # hide hidden files?
                if elem.startswith('.') and hide_hidden:
                    continue

                # excluded by filters?
144
                if filter and bool(filter.search(elem)) != reverse:
145
                    continue
146
                
Gero Müller's avatar
Gero Müller committed
147
148
149
                info = {
                    'name': elem
                }
150

Gero Müller's avatar
Gero Müller committed
151
                fullpath = os.path.join(path_expand, elem)
Gero Müller's avatar
Gero Müller committed
152
                stats = os.lstat(fullpath)
Gero Müller's avatar
Gero Müller committed
153
                is_symlink = stat.S_ISLNK(stats.st_mode)
Gero Müller's avatar
Gero Müller committed
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
                if is_symlink:
                    realfullpath = os.path.realpath(fullpath)
                    info.update({'symlink': True, 'realpath': realfullpath})
                    if os.path.exists(realfullpath):
                        stats = os.stat(realfullpath)

                info.update({
                    'size': stats.st_size,
                    'mtime': stats.st_mtime,
                    'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
                })

                filelist.append(info)
                if deep:
                    filelist.extend(self.get_file_list(fullpath, deep,
                                                       filter, reverse))
        except:
            pass
172
173

        # Determine the parent
Gero Müller's avatar
Gero Müller committed
174
        # parentpath = path_expand[:-1] if path_expand.endswith(os.sep) and
Gero Müller's avatar
Gero Müller committed
175
        #    path_expand != os.sep else path_expand
176
        parentpath = os.path.dirname(path_expand)
177
178
179
        data = {'filelist': filelist, 'parentpath': parentpath}
        if encode_json:
            return json.dumps(data)
180
181
        else:
            return data
182

Gero Müller's avatar
Gero Müller committed
183
184
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
185
186
187
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
188
189
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
190
            # dir case
Marcel's avatar
Marcel committed
191
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
192
193
194
195
196
197
198
199
200
201
202
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
203
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
204
205
206
207
208
209
210
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
211
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
212
213
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
214
215
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
216
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
217
218
219
220
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
221
222
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
223
224
225
                suggestion += '/'
            suggestions.append(suggestion)

226
        return suggestions if not encode_json else json.dumps(suggestions)
227

murban's avatar
murban committed
228
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
229
        path = self.expand(path)
murban's avatar
murban committed
230
        path = path[1:] if path.startswith(os.sep) else path
231
232
        if path == "":
            return path
murban's avatar
murban committed
233
        path = path[:-1] if path.endswith(os.sep) else path
234
235
        return path

murban's avatar
murban committed
236
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
237
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
238
239
        name = self.expand(name)

240
        # folder with the same name existent?
241
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
242
        fullpath = os.path.join(path, name)
243
244
245
        try:
            os.mkdir(fullpath)
        except Exception as e:
246
            # raise Exception("You don't have the permission to create this folder!")
247
248
            raise Exception(str(e))

murban's avatar
murban committed
249
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
250
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
251
252
        name = self.expand(name)

253
        # file with the same name existent?
254
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
255
        fullpath = os.path.join(path, name)
256
257
258
259
260
261
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

262
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
263
264
265
266
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

267
268
269
270
271
272
273
        try:
            if force == False:
                new_name = self.handle_file_name_collision(new_name, path)
            name = os.path.join(path, name)
            new_name = os.path.join(path, new_name)
            os.renames(name, new_name)
            return
Gero Müller's avatar
Gero Müller committed
274
        except Exception as e:
275
            return str(e)
276

murban's avatar
murban committed
277
278
    def remove(self, path):
        if isinstance(path, list):
279
280
281
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
282
283
284
285
286
            path = self.expand(path)
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)
287
288

    def compress(self, path, paths, name):
Gero Müller's avatar
Gero Müller committed
289
290
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
291
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
292
293
294

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
295

Gero Müller's avatar
Gero Müller committed
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
316
317
                            ap = fullp[
                                len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
318
319
320
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
321
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
322
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
323
                    archive.write(p, ap)
324

325
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
326
        # TODO
327
328
329
330
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
331
                self.paste(path, p, cut)
332
333
            return True

334
335
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
        logger.error("")
Gero Müller's avatar
Gero Müller committed
336
337
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
338
        fulltarget = os.path.join(path, target)
339

340
341
        if os.path.isdir(fullsrc):
            shutil.copytree(fullsrc, fulltarget)
342
            if cut:
343
                shutil.rmtree(fullsrc)
344
        else:
345
            shutil.copy2(fullsrc, fulltarget)
346
            if cut:
347
                os.remove(fullsrc)
348

349
350
    def save_file(self, path, content, force=True, binary=False,
                   utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
351
        path = self.expand(path)
Marcel's avatar
Marcel committed
352
        # check if file already exists
murban's avatar
murban committed
353
        if os.path.exists(path) and not force:
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
            mtime  = 0
        
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
371
        else:
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
            watch_error = ""
        
        return json.dumps({
            "mtime": os.path.getmtime(path),
            "success": mtime > 0,
            "watch_error": watch_error
        })
    
    def get_file(self, path, binary=False,
                  utf8=False, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
        
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
            mtime = os.path.getmtime(path)
        except Exception as e:
            mtime  = 0
            content = ""
        
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
            "watch_error": watch_error
        })
406

Gero Müller's avatar
Gero Müller committed
407
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
408
                          path=None, force=True, append=False):
409
410
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Gero Müller's avatar
Gero Müller committed
411
412
413
414
415
416
        #    return False, "Permission denied!"

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

417
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
418
419
420
421
422
423
424
425
426
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

murban's avatar
murban committed
427
    def get_file_content(self, path):
Marcel's avatar
Marcel committed
428
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
429
        f = open(path, "rb")
430
431
        content = f.read()
        f.close()
Marcel's avatar
Marcel committed
432
        return content
433

Marcel's avatar
Marcel committed
434
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
435
        path = self.expand(path)
Marcel's avatar
Marcel committed
436
437
        return os.path.getmtime(path)

murban's avatar
murban committed
438
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
439
        path = self.expand(path)
440
        extension = path.split(".")[-1]
441
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
442

murban's avatar
murban committed
443
    def handle_file_name_collision(self, name, path):
444
        # collision?
murban's avatar
murban committed
445
        files = os.listdir(path)
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
        counter = prename.split("_")[-1]

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
                preprename = "_".join(prename.split("_")[:-1])
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
Gero Müller's avatar
Gero Müller committed
476
477
478
            newname = "%s_%d%s" % (preprename,
                                   counter,
                                   "" if extension == "" else "." + extension)
479
        else:
Gero Müller's avatar
Gero Müller committed
480
481
            newname = "%s_1%s" % (
                prename, "" if extension == "" else "." + extension)
482
483

        # return
murban's avatar
murban committed
484
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
485

Marcel's avatar
Marcel committed
486
487
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
488

Gero Müller's avatar
Gero Müller committed
489
    def thumbnail(self, path, width=100, height=100, sharpen=True):
490
491
492
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
493
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
494
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
495
            if sharpen:
496
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
497
            img.save(output, "JPEG")
498
499
500
501
502
503
504
505
506
507
508
509
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

510
511
512
513
    def _monitor_callback(self, event):
        if event.action_name == 'access':
            return
        
514
515
516
517
518
        if os.path.exists(event.watch.path):
            mtime = os.path.getmtime(event.watch.path)
        else:
            mtime = -1
        
519
520
521
522
523
524
525
526
527
        with self._monitor_lock:
            # inform all listener
            for combined_id in event.watch.listener:
                vispa.remote.send_topic(
                    "extension.%s.socket.watch" % combined_id[1],
                    window_id=combined_id[0],
                    data={
                        "path": event.watch.path,
                        "action_name": event.action_name,
528
529
                        "watch_id": combined_id[2],
                        "mtime": mtime
530
531
532
                    })
            
            # cleanup yourself
533
            if event.action_name in ['delete self','move self']:
534
535
                for combined_id in event.watch.listener:
                    del self._monitor_listener[combined_id]
536
                event.watch.listener = []
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
                self._check_watch(event.watch)
    
    def _check_watch(self, watch):
        if len(watch.listener) == 0:
            del self._monitor_watches[watch.path]
            self._monitor_thread.remove_watch(watch)
        
    def watch(self, path, window_id, view_id, watch_id):
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
        combined_id = (window_id, view_id, watch_id)
        
        with self._monitor_lock:
            # watch is in use
            if combined_id in self._monitor_listener:
                # has path changed?
                if self._monitor_listener[combined_id].path == path:
                    return ""
                else:
                    self._monitor_listener[combined_id].listener.remove(combined_id)
                    self._check_watch(self._monitor_listener[combined_id])
                    del self._monitor_listener[combined_id]
            
            # create or get the desired watch
            if path not in self._monitor_watches:
                if os.path.isfile(path):
                    watch = self._monitor_thread.add_file_watch(path)
                elif os.path.isdir(path):
                    watch = self._monitor_thread.add_dir_watch(path)
                else:
                    return "This kind of file can't be wachted"
                watch.listener = []
                self._monitor_watches[path] = watch
            else:
                watch = self._monitor_watches[path]
            
            if combined_id in watch.listener:
                return "This file is already watched" # this should never occur
            
            watch.listener.append(combined_id)
            self._monitor_listener[combined_id] = watch

        return ""
        
    def unwatch(self, window_id, view_id, watch_id=None):
        with self._monitor_lock:
            if watch_id is None:
                # will perform poor with lots of watches/listener
                for path, watch in self._monitor_watches.items():
                    for combined_id in watch.listener:
                        if window_id==combined_id[0] and view_id==combined_id[1]:
                            watch.listener.remove(combined_id)
                            del self._monitor_listener[combined_id]
                    
                    self._check_watch(watch)
            
            else:
                combined_id = (window_id, view_id, watch_id)
                if combined_id in self._monitor_listener:
                    self._monitor_listener[combined_id].listener.remove(combined_id)
                    self._check_watch(self._monitor_listener[combined_id])
                    del self._monitor_listener[combined_id]

        return ""

604

Marcel's avatar
Marcel committed
605
606
607
608
609
610
611
612
def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1

613

Marcel's avatar
Marcel committed
614
615
616
617
618
619
620
621
622
def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1