server.py 16.1 KB
Newer Older
1
2
# -*- coding: utf-8 -*-

3
4
5
from logging import config as loggingcfg
from pkgutil import iter_modules
import importlib
6
7
8
9
import inspect
import logging
import os
import sys
10

Gero Müller's avatar
Gero Müller committed
11
from sqlalchemy.orm import scoped_session, sessionmaker
12
from vispa.models import Base as sql_base
13
from vispa.models.extension import *
14
from vispa.models.preference import *
15
16
from vispa.models.user import *
from vispa.models.workspace import *
17
18
19
20
21
import cherrypy
import sqlalchemy
import vispa.extensions
import vispa.plugins.template
import vispa.url
22

23
24
from vispa.models.shortcuts import *
import vispa.models.alembic
Gero Müller's avatar
Gero Müller committed
25
from urlparse import urlparse
26
27
28
29
logger = logging.getLogger(__name__)


class AbstractExtension(object):
Gero Müller's avatar
Gero Müller committed
30

31
32
33
34
35
    """ Base class for Extensions """

    def __init__(self, server):
        self.server = server

36
37
38
39
40
    def name(self):
        """
        Return the name of the Extension.
        This name is used as part of the URL.
        """
41
42
43
        raise NotImplementedError

    def dependencies(self):
44
45
46
        """
        Return a list of Extension names this Extension depends on.
        """
47
48
        raise NotImplementedError

49
50
51
52
    def setup(self):
        """
        Setup the extension.
        """
53
        raise NotImplementedError
54

55
56
57
    def config(self):
        return {}

58
59
60
61
    def add_controller(self, controller):
        """
        Mount a CherryPy controller using the extension name for path.

Gero Müller's avatar
Gero Müller committed
62
        :param controller: filename relative to extension directory
63
        """
64
        controller.extension = self
65
        self.server.controller.mount_extension_controller(self.name(),
66
67
                                                          controller)

68
69
70
71
    def add_workspace_directoy(self, directory="workspace"):
        """
        Add files to be transferred to the worker.

Gero Müller's avatar
Gero Müller committed
72
        :param directoy: directory relative to extension directory
73
        """
74
        class_dir = os.path.dirname(inspect.getabsfile(self.__class__))
75
76
77
        local = os.path.join(class_dir, directory)
        remote = os.path.join('vispa', 'extensions', self.name(), directory)
        vispa.workspace.add_directory_files(local, remote)
78
79
80
        self.__workspace_directory = directory

    def get_workspace_instance(self, name, key=None, user=None,
81
                               workspace=None, db=None, **kwargs):
82
        classname = ".".join(["vispa.extensions", self.name(),
Marcel Rieger's avatar
Marcel Rieger committed
83
                              self.__workspace_directory, name])
84
        return vispa.workspace.get_instance(classname, key, user,
85
                                            workspace, db, init_args=kwargs)
86
87

    def clear_workspace_instance(self, name, key=None, user=None,
Marcel Rieger's avatar
Marcel Rieger committed
88
                                 workspace=None, db=None):
89
        classname = ".".join(["vispa.extensions", self.name(),
Marcel Rieger's avatar
Marcel Rieger committed
90
                              self.__workspace_directory, name])
91
        return vispa.workspace.clear_instance(classname, key, user,
Marcel Rieger's avatar
Marcel Rieger committed
92
                                              workspace, db)
Marcel's avatar
Marcel committed
93

94
95
96
97
    def create_topic(self, topic="", view_id=None):
        if view_id is None:
            view_id = cherrypy.request.private_params["_viewId"]
        return "extension.%s.socket.%s" % (view_id, topic)
98

Gero Müller's avatar
Gero Müller committed
99

100
class Server(object):
Marcel's avatar
Marcel committed
101

Gero Müller's avatar
Gero Müller committed
102
103
104
    __default_server_config = {
        'server.socket_host': '127.0.0.1',
        'server.socket_port': 4282,
Gero Müller's avatar
Gero Müller committed
105
106
        'server.socket_queue_size': 10,
        'server.request_queue_size': 10,
Gero Müller's avatar
Gero Müller committed
107
        'server.thread_pool': 10,
108
        'engine.autoreload.on': False
Gero Müller's avatar
Gero Müller committed
109
110
    }

111
    @property
112
    def __default_mount_config(self):
Gero Müller's avatar
Gero Müller committed
113
114
        base_dynamic = vispa.url.dynamic('/')
        base_static = vispa.url.static('/', timestamp=False)
115
116
        return {
            '/': {
Gero Müller's avatar
Gero Müller committed
117
                'tools.proxy.on': False,
118
                'tools.encode.on': False,
119
                'tools.db.on': True,
120
                'tools.private_parameters.on': True,
121
                'tools.user.on': True,
Gero Müller's avatar
Gero Müller committed
122
                'tools.user.redirect_url': vispa.url.dynamic('/login'),
123
                'tools.user.redirect_url_reverse': vispa.url.dynamic('/'),
124
                'tools.workspace.on': False,
Gero Müller's avatar
Gero Müller committed
125
                'tools.sessions.on': True,
Gero Müller's avatar
Gero Müller committed
126
                'tools.sessions.path': urlparse(base_dynamic).path,
127
128
129
                'tools.sessions.storage_type': 'file',
                'tools.sessions.storage_path': vispa.datapath('sessions'),
                'tools.sessions.timeout': 1440,
Gero Müller's avatar
Gero Müller committed
130
                'tools.staticdir.on': False,
131
                'tools.gzip.on': True,
132
                'tools.gzip.mime_types': ['text/html', 'text/css',
Gero Müller's avatar
Gero Müller committed
133
134
                                          'application/x-javascript',
                                          'application/json'],
135
136
                'tools.render.common_data': {
                    'base_dynamic': base_dynamic,
137
138
                    'base_static': base_static,
                    'dev_mode': vispa.config("web", "dev_mode", True)
139
                },
Gero Müller's avatar
Gero Müller committed
140
                "tools.status.on": True,
141
            },
142
            '/extensions': {
143
144
                'tools.workspace.on': True,
                'tools.ajax.on': True
145
            },
146
147
148
            '/fs': {
                'tools.workspace.on': True
            },
149
150
151
            '/ajax': {
                'tools.workspace.on': True,
                'tools.ajax.on': True
Marcel Rieger's avatar
Marcel Rieger committed
152
153
154
155
156
157
            },
            '/error': {
                'tools.db.on': False,
                'tools.private_parameters.on': False,
                'tools.user.on': False,
                'tools.workspace.on': False
158
            }
Gero Müller's avatar
Gero Müller committed
159
        }
160

Gero Müller's avatar
Gero Müller committed
161
162
163
164
165
166
    def __init__(self, **kwargs):
        self.__init_paths(**kwargs)
        self.__init_database(**kwargs)
        self.__init_tools(**kwargs)
        self.__init_platform(**kwargs)
        self.__init_plugins(**kwargs)
167

Gero Müller's avatar
Gero Müller committed
168
    def __init_paths(self, **kwargs):
169
        # dir for variable files and folders
Gero Müller's avatar
Gero Müller committed
170
        self.var_dir = os.path.abspath(kwargs.get('vardir', ''))
171
        if not os.path.exists(self.var_dir):
172
173
            os.makedirs(self.var_dir)

174
        logger.info('Using %s as data dir.' % self.var_dir)
175
        vispa.set_datapath(self.var_dir)
176

177
        # log dir
Marcel's avatar
Marcel committed
178
        self.log_dir = os.path.join(self.var_dir, 'logs')
179
180
        if not os.path.exists(self.log_dir):
            os.mkdir(self.log_dir)
181

182
        # session dir
Marcel's avatar
Marcel committed
183
        self.session_dir = os.path.join(self.var_dir, 'sessions')
184
185
        if not os.path.exists(self.session_dir):
            os.mkdir(self.session_dir)
186

187
188
189
190
        # dir for workspace files
        self.workspace_dir = os.path.join(self.var_dir, 'workspace')
        if not os.path.exists(self.workspace_dir):
            os.mkdir(self.workspace_dir)
191

192
        # cache dir
Marcel's avatar
Marcel committed
193
        self.cache_dir = os.path.join(self.var_dir, 'cache')
194
195
        if not os.path.exists(self.cache_dir):
            os.mkdir(self.cache_dir)
196

197
        # conf dir
Gero Müller's avatar
Gero Müller committed
198
        self.conf_dir = os.path.abspath(kwargs.get('configdir', ''))
199
        logger.info('Using %s as config dir.' % self.conf_dir)
200
201
        vispa.set_configpath(self.conf_dir)
        vispa.config.read(vispa.configpath('vispa.ini'))
202

Gero Müller's avatar
Gero Müller committed
203
    def __init_database(self, **kwargs):
204
205
        sa_identifier = vispa.config('database', 'sqlalchemy.url',
                                     'sqlite:///%s/vispa.db' % self.var_dir)
206
207
208
209
210
        pool_size = int(vispa.config('database', 'sqlalchemy.pool_size', '10'))
        max_overflow = int(vispa.config(
            'database',
            'sqlalchemy.max_overflow',
            '10'))
211
212
213
214
215
216
        # https://github.com/mitsuhiko/flask-sqlalchemy/issues/2
        # http://docs.sqlalchemy.org/en/latest/core/pooling.html#dealing-with-disconnects
        pool_recycle = int(vispa.config(
            'database',
            'sqlalchemy.pool_recycle',
            '7200'))
217
        logger.info('Use database %s.' % sa_identifier)
218
219
220
221
222
        try:
            self._engine = sqlalchemy.create_engine(
                sa_identifier,
                echo=False,
                pool_size=pool_size,
223
                pool_recycle=pool_recycle,
224
225
226
227
228
                max_overflow=max_overflow)
        except TypeError:
            self._engine = sqlalchemy.create_engine(
                sa_identifier,
                echo=False)
229

Gero Müller's avatar
Gero Müller committed
230
231
        if vispa.config('alembic', 'use_alembic',
                        not sa_identifier.startswith('sqlite')):
232
233
234
            logger.info("Use alembic")
            if vispa.config('alembic', 'auto_migrate', True):
                vispa.models.alembic.migrate(self._engine)
235
        else:
236
237
            logger.info("Do not use alembic")
            sql_base.metadata.create_all(self._engine)
238

Gero Müller's avatar
Gero Müller committed
239
    def __init_plugins(self, **kwargs):
Gero Müller's avatar
Gero Müller committed
240
        logger.info("init plugins")
Gero Müller's avatar
Gero Müller committed
241
        if vispa.config('websockets', 'enabled', False):
242
            from ws4py.server.cherrypyserver import WebSocketPlugin
Gero Müller's avatar
Gero Müller committed
243
            WebSocketPlugin(cherrypy.engine).subscribe()
Gero Müller's avatar
Gero Müller committed
244
245

        logger.info("create bus")
246
247
        from vispa.socketbus import Bus
        vispa.bus = Bus()
248

Gero Müller's avatar
Gero Müller committed
249
        logger.info("setup templates")
250
        mako_lookup_dir = os.path.join(os.path.dirname(__file__), "templates")
251
        vispa.plugins.template.MakoPlugin(cherrypy.engine,
252
                                          base_dir=mako_lookup_dir,
253
                                          module_dir=self.cache_dir
Gero Müller's avatar
Gero Müller committed
254
                                          ).subscribe()
Gero Müller's avatar
Gero Müller committed
255

Gero Müller's avatar
Gero Müller committed
256
    def __init_tools(self, **kwargs):
257
258
259
260
        from vispa.tools.template import MakoTool
        cherrypy.tools.render = MakoTool()

        from vispa.tools.db import SqlAlchemyTool
261
        cherrypy.tools.db = SqlAlchemyTool(self._engine)
262
263
264
265
266
267
268
269
270
271
272
273
274

        from vispa.tools.user import UserTool
        cherrypy.tools.user = UserTool()

        from vispa.tools.workspace import WorkspaceTool
        cherrypy.tools.workspace = WorkspaceTool()

        from vispa.tools.parameters import PrivateParameterFilter
        cherrypy.tools.private_parameters = PrivateParameterFilter()

        from vispa.tools.device import DeviceTool
        cherrypy.tools.device = DeviceTool()

Gero Müller's avatar
Gero Müller committed
275
276
277
        from vispa.tools.ajax import AjaxTool
        cherrypy.tools.ajax = AjaxTool()

278
279
280
        from vispa.tools.method import MethodTool
        cherrypy.tools.method = MethodTool()

Gero Müller's avatar
Gero Müller committed
281
282
        from vispa.tools.json_parameters import JsonParameters
        cherrypy.tools.json_parameters = JsonParameters()
283

Gero Müller's avatar
Gero Müller committed
284
285
286
        from vispa.tools.status import StatusMonitor
        cherrypy.tools.status = StatusMonitor()

287
        try:
288
            from ws4py.server.cherrypyserver import WebSocketTool
289
290
291
292
            cherrypy.tools.websocket = WebSocketTool()
        except:
            pass

Gero Müller's avatar
Gero Müller committed
293
    def __init_platform(self, **kwargs):
Gero Müller's avatar
Gero Müller committed
294
        cherrypy.config.update(self.__default_server_config)
Gero Müller's avatar
Gero Müller committed
295

296
        cherrypy_conf = vispa.configpath('cherrypy.ini')
Gero Müller's avatar
Gero Müller committed
297
298
299
        if os.path.isfile(cherrypy_conf):
            cherrypy.config.update(cherrypy_conf)

Gero Müller's avatar
Gero Müller committed
300
301
302
        port = kwargs.get('port', None)
        if port:
            cherrypy.config.update({'server.socket_port': int(port)})
Gero Müller's avatar
Gero Müller committed
303

Gero Müller's avatar
Gero Müller committed
304
        logger.info("create root controller")
305
        from vispa.controller.root import RootController
306
        self.controller = RootController(self)
Gero Müller's avatar
Gero Müller committed
307
        logger.info("load extensions")
308
        self._load_extensions()
309
        script_name = vispa.url.dynamic('/', encoding='utf-8')
310

Gero Müller's avatar
Gero Müller committed
311
        logger.info("mount app")
312
        app_config = self.__default_mount_config
313

314
315
        # merge extension config into app config
        for extension in self._extensions.values():
316
317
318
319
320
            # config is structured for multiple mountpoints
            for mount, conf in extension.config().items():
                key = "/extensions/%s%s" % (extension.name(), "" if mount == "/" else mount)
                app_config[key] = conf

321
322
        self.__application = cherrypy.tree.mount(self.controller, script_name, app_config)

Gero Müller's avatar
Gero Müller committed
323
        if os.path.isfile(cherrypy_conf):
Gero Müller's avatar
Gero Müller committed
324
            self.__application.merge(cherrypy_conf)
325

326
327
328
329
330
331
332
333
    def _on_exit(self):
        vispa.bus.send_topic("exit", broadcast=True)
        vispa.publish("exit")

    def _on_stop(self):
        vispa.bus.send_topic("stop", broadcast=True)
        vispa.publish("stop")

Gero Müller's avatar
Gero Müller committed
334
    def start(self):
Gero Müller's avatar
Gero Müller committed
335
336
        if hasattr(cherrypy.engine, 'signal_handler'):
            cherrypy.engine.signal_handler.subscribe()
337
338
        cherrypy.engine.subscribe("exit", self._on_exit, 0)
        cherrypy.engine.subscribe("stop", self._on_stop, 0)
Gero Müller's avatar
Gero Müller committed
339
        cherrypy.engine.start()
Gero Müller's avatar
Gero Müller committed
340
        vispa.setup_thread_dump()
Gero Müller's avatar
Gero Müller committed
341
342
343

    def run(self):
        self.start()
Marcel Rieger's avatar
Marcel Rieger committed
344
        cherrypy.engine.block()
345
346
347
348
349
350
351
352

    def _load_extensions(self):
        self._extensions = {}
        # loop through all extensions and import their files
        # so that 'AbstractExtension' will know about its subclasses
        modulenames = []
        ignored = vispa.config('extensions', 'ignore', [])
        extensionspath = vispa.datapath('extensions')
353
        vispa.extensions.__path__.append(extensionspath)
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
        for _, modulename, ispkg in iter_modules(vispa.extensions.__path__):
            if not ispkg:
                continue
            if modulename in ignored:
                logger.info('Ignore extension: %s ' % modulename)
                continue
            try:
                importlib.import_module('vispa.extensions.%s' % modulename)
                modulenames.append(modulename)
            except:
                _, message, _ = sys.exc_info()
                logger.warning('Exception importing extension %s: %s' %
                               (modulename, message))
                vispa.log_exception()

        for _, modulename, ispkg in iter_modules():
            if not ispkg:
                continue
            if not modulename.startswith('vispa_'):
                logger.debug('Not a vispa extension: %s ' % modulename)
                continue
            if modulename in ignored:
                logger.info('Ignore extension: %s ' % modulename)
                continue
            try:
                importlib.import_module(modulename)
                modulenames.append(modulename)
            except:
                _, message, _ = sys.exc_info()
                logger.warning('Exception importing extension %s: %s'
                               % (modulename, message))
                vispa.log_exception()

387
388
        load = [x.strip() for x in
                vispa.config('extensions', 'import', '').split(',')]
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
        load = filter(lambda x: len(x) > 0, load)
        for ext in load:
            logger.info('Import extension: %s ' % ext)
            try:
                importlib.import_module(ext)
                modulenames.append(ext.split('.')[-1])
            except:
                _, message, _ = sys.exc_info()
                logger.warning('Exception importing extension %s: %s'
                               % (modulename, message))
                vispa.log_exception()

        # instantiate all subclasses of the 'AbstractExtension'
        dependencies = {}
        for cls in AbstractExtension.__subclasses__():
            # check: is the modulename accepted?
            modulename = cls.__module__.split('.')[-1]
            if modulename in ignored:
                continue

            logger.debug('Loading Extension "%s"' % cls)
            extension = cls(self)
411
            name = extension.name()
412
            if name in self._extensions.keys():
413
414
                raise Exception("Fail to load extension: '%s'."
                                "It already exsits!" % name)
415
416
417
418
419
420
421
422
423
424
425
            self._extensions[name] = extension
            # update the 'dependencies' dict
            for dep in extension.dependencies():
                if dep not in dependencies.keys():
                    dependencies[dep] = [name]
                else:
                    dependencies[dep].append(name)

        # check dependencies
        for dep in dependencies.keys():
            if dep not in self._extensions:
426
427
428
429
                data = dep, ", ".join(dependencies[dep])
                msg = """"The following dependency could not be found: '%s'.
                The following extensions depend on it: %s""" % data
                raise Exception(msg)
430
431
432

        # setup all valid extenions
        for extension in self._extensions.values():
433
434
435
            try:
                extension.setup()
            except:
Gero Müller's avatar
Gero Müller committed
436
437
438
                logger.warn(
                    "Extension '%s' failed to setup!" %
                    extension.name())
439
                vispa.log_exception()
440
441
442
443
444
445

    def extension(self, name):
        return self._extensions[name]

    def extensions(self):
        return self._extensions.values()
Gero Müller's avatar
Gero Müller committed
446
447
448

    def application(self):
        return self.__application