__init__.py 13.5 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

3
4
5
"""
Basic functionality for the VISPA platform
"""
Marcel's avatar
Marcel committed
6

7
8
9
10
11
from StringIO import StringIO
from fileinput import filename
import csv
import inspect
import logging
Marcel's avatar
Marcel committed
12
import os
Gero Müller's avatar
Gero Müller committed
13
import re
14
15
import signal
import smtplib
Gero Müller's avatar
Gero Müller committed
16
import socket
17
18
19
import sys
import tempfile
import thread
Gero Müller's avatar
Gero Müller committed
20
21
import token
import tokenize
22
import traceback
Gero Müller's avatar
Gero Müller committed
23
import copy
24
25

import cherrypy
26
from cherrypy.lib.httputil import response_codes
27

28
29
from . import url

Gero Müller's avatar
Gero Müller committed
30
31
32
33
from smtplib import SMTP
from email.MIMEText import MIMEText
from email.Header import Header
from email.Utils import parseaddr, formataddr
Gero Müller's avatar
Gero Müller committed
34

35
try:
Gero Müller's avatar
Gero Müller committed
36
    import ConfigParser as cp
37
except:
38
    import configparser as cp
39

Gero Müller's avatar
Gero Müller committed
40
41
logger = logging.getLogger(__name__)

Gero Müller's avatar
Gero Müller committed
42
43
44
45
46
# http://stackoverflow.com/a/24517154
from .version import __version__
VERSION = __version__
RELEASE_VERSION, _, BUGFIX_VERSION = __version__.rpartition(".")
# http://semver.org/
Gero Müller's avatar
Gero Müller committed
47
MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION = __version__.split(".")
48

49
50
_datapath = 'data'
_configpath = 'conf'
51
52
53
_codepath = os.path.dirname(inspect.getfile(inspect.currentframe()))
_codepath = os.path.dirname(_codepath)

Marcel's avatar
Marcel committed
54

55
def datapath(*args):
56
57
58
    """
    returns the path relative to the datapath
    """
59
    return os.path.join(_datapath, *args)
60

61

62
63
64
65
def set_datapath(p):
    global _datapath
    _datapath = p

66

67
def configpath(*args):
68
    return os.path.join(_configpath, *args)
69

70

71
72
73
74
def set_configpath(p):
    global _configpath
    _configpath = os.path.abspath(p)

75

76
def codepath(*args):
77
    return os.path.join(_codepath, *args)
78

79

80
81
82
def set_codepath(p):
    global _codepath
    _codepath = p
Marcel's avatar
Marcel committed
83

84

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
def config_files(directory):
    conf_d_dir = configpath(directory)
    if os.path.isdir(conf_d_dir):
        config_files = [f for f in os.listdir(conf_d_dir) if f.endswith(".ini") and os.path.isfile(os.path.join(conf_d_dir, f))]
        config_files.sort()
        return [os.path.join(conf_d_dir, f) for f in config_files]
    else:
        return []
            
def setup_config(conf_dir): 
    logger.info('Using %s as config dir.' % conf_dir)
    set_configpath(conf_dir)

    logger.info('Read config file: %s', configpath('vispa.ini'))
    config.read(configpath('vispa.ini'))

    # load all ini files
    _config_files = config_files('vispa.d')
    logger.info('Read config files: %s', _config_files)
    config.read(_config_files)


107
class VispaConfigParser(cp.SafeConfigParser):
Gero Müller's avatar
Gero Müller committed
108
    def __call__(self, section, option, default=None):
109
110
        if self.has_option(section, option):
            value = self.get(section, option)
111
112
113
            # try to cast according to 'default'
            if default is None:
                return value
Gero Müller's avatar
Gero Müller committed
114
115
116
117
118
119
            # boolean
            if isinstance(default, bool):
                if value == 'True':
                    return True
                elif value == 'False':
                    return False
120
121
122
123
124
125
            # int
            if isinstance(default, int):
                return int(value)
            # long
            if isinstance(default, long):
                return long(value)
126
127
128
129
            # string
            if isinstance(default, str):
                return str(value)
            # list
Gero Müller's avatar
Gero Müller committed
130
            if isinstance(default, list):
Gero Müller's avatar
Gero Müller committed
131
132
133
134
135
136
137
138
139
                l = []
                g = tokenize.generate_tokens(StringIO(value).readline)
                for toknum, tokval, _, _, _  in g:
                    if toknum == token.NAME or toknum == token.NUMBER:
                        l.append(tokval)
                    elif toknum == token.STRING:
                        l.append(tokval[1:-1])
                return l

140
            # tuple
Gero Müller's avatar
Gero Müller committed
141
            if isinstance(default, tuple):
142
143
144
145
146
                if len(value):
                    return tuple(value.split(','))
                else:
                    return ()
            # TODO: dict
Gero Müller's avatar
Gero Müller committed
147
            return value
Gero Müller's avatar
Gero Müller committed
148
149
150
        else:
            return default

Marcel's avatar
Marcel committed
151
# setup a config parser for "vispa.ini"
152
config = VispaConfigParser()
153

Marcel's avatar
Marcel committed
154
# a little publish/subscribe system
155
156
_callbacks = {}

157

Marcel's avatar
Marcel committed
158
159
160
161
def subscribe(topic, callback):
    if callback not in _callbacks.get(topic, []):
        _callbacks.setdefault(topic, []).append(callback)
register_callback = subscribe
162

163

Marcel's avatar
Marcel committed
164
165
166
def publish(topic, *args, **kwargs):
    values = []
    for callback in _callbacks.get(topic, []):
167
        try:
Marcel's avatar
Marcel committed
168
169
            value = callback(*args, **kwargs)
            values.append(value)
170
        except Exception:
Marcel's avatar
Marcel committed
171
            values.append(None)
172
            log_exception()
Marcel's avatar
Marcel committed
173
174
175
    return values
fire_callback = publish

176

Gero Müller's avatar
Gero Müller committed
177
def exception_string():
Marcel's avatar
Marcel committed
178
179
    exc_type, exc_value, exc_tb = sys.exc_info()
    st = traceback.format_exception(exc_type, exc_value, exc_tb)
180
    return  ''.join(st)
Gero Müller's avatar
Gero Müller committed
181

182

Gero Müller's avatar
Gero Müller committed
183
184
def log_exception():
    sys.stderr.write(exception_string())
Gero Müller's avatar
Gero Müller committed
185
186


187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
class Netstat(object):
    """
    Parses /dev/net/tcp to determine which user owns the specified port
    """

    _proc_net_tcp_re = re.compile("\W+([0-9]+):\W+([0-9A-F]+):([0-9A-F]+)"
                                  "\W+([0-9A-F]+):([0-9A-F]+)\W+([0-9A-F]+)"
                                  "\W+([0-9A-F]+):([0-9A-F]+)"
                                  "\W+([0-9A-F]+):([0-9A-Z]+)"
                                  "\W+([0-9A-F]+)\W+([0-9A-F]+)"
                                  "\W+([0-9A-F]+).*")

    _IDX_LOCAL_IP = 2
    _IDX_LOCAL_PORT = 3
    _IDX_REMOTE_IP = 4
    _IDX_REMOTE_PORT = 5
    _IDX_USERID = 12
Gero Müller's avatar
Gero Müller committed
204
205

    @staticmethod
206
    def _ipv4_to_hex(ip):
Gero Müller's avatar
Gero Müller committed
207
208
209
210
211
        parts = ip.split(".")
        parts.reverse()
        return "".join(["%02X" % int(part) for part in parts])

    @staticmethod
212
    def _port_to_hex(port):
Gero Müller's avatar
Gero Müller committed
213
        return "%02X" % port
214

Gero Müller's avatar
Gero Müller committed
215
    @staticmethod
216
217
218
    def _get_socket_owner_file(filename, local_ip, local_port,
                               remote_ip, remote_port):
        f = open(filename, 'r')
Gero Müller's avatar
Gero Müller committed
219
220
221
222
223
        userid = None
        for line in f:
            m = Netstat._proc_net_tcp_re.match(line)
            if not m:
                continue
224
            if m.group(Netstat._IDX_LOCAL_IP) != local_ip:
Gero Müller's avatar
Gero Müller committed
225
                continue
226
            if m.group(Netstat._IDX_LOCAL_PORT) != local_port:
Gero Müller's avatar
Gero Müller committed
227
                continue
228
            if m.group(Netstat._IDX_REMOTE_IP) != remote_ip:
Gero Müller's avatar
Gero Müller committed
229
                continue
230
            if m.group(Netstat._IDX_REMOTE_PORT) != remote_port:
Gero Müller's avatar
Gero Müller committed
231
                continue
232
            userid = int(m.group(Netstat._IDX_USERID))
Gero Müller's avatar
Gero Müller committed
233
234
235
            break
        f.close()
        return userid
236

Gero Müller's avatar
Gero Müller committed
237
    @staticmethod
238
    def _get_socket_owner_ipv4(local_ip, local_port, remote_ip, remote_port):
Gero Müller's avatar
Gero Müller committed
239
240
241
242
        local_port = Netstat.port_to_hex(local_port)
        local_ip = Netstat.ipv4_to_hex(local_ip)
        remote_port = Netstat.port_to_hex(remote_port)
        remote_ip = Netstat.ipv4_to_hex(remote_ip)
243
244
245
246
        return Netstat.get_socket_owner_file('/proc/net/tcp',
                                             local_ip, local_port,
                                             remote_ip, remote_port)

Gero Müller's avatar
Gero Müller committed
247
248
    @staticmethod
    def get_socket_owner(local_ip, local_port, remote_ip, remote_port):
249
250
251
252
253
254
255
256
257
258
        """
        Returns the system id of the local user which established the
        specifiec connection

        :param local_ip: ip address of the local peer
        :param local_ip: port the local peer
        :param remote_ip: ip address of the remote peer
        :param remote_ip: port the remote peer
        :rtype: user id of the user who opened this port or None
        """
Gero Müller's avatar
Gero Müller committed
259
        # prepare adresses
260
261
        _locals = socket.getaddrinfo(local_ip, local_port,
                                     0, 0, socket.SOL_TCP)
Gero Müller's avatar
Gero Müller committed
262
        # [(2, 1, 6, '', ('127.0.0.1', 4282))]
263
264
        _remotes = socket.getaddrinfo(remote_ip, remote_port,
                                      0, 0, socket.SOL_TCP)
Gero Müller's avatar
Gero Müller committed
265
        # [(2, 1, 6, '', ('127.0.0.1', 53726))]
266

Gero Müller's avatar
Gero Müller committed
267
268
269
270
271
272
273
        for local in _locals:
            for remote in _remotes:
                if remote[0] != local[0]:
                    continue
                if remote[4][0] != local[4][0]:
                    continue
                if remote[0] == socket.AF_INET:
274
275
276
277
                    return Netstat.get_socket_owner_ipv4(local[4][0],
                                                         local[4][1],
                                                         remote[4][0],
                                                         remote[4][1])
Gero Müller's avatar
Gero Müller committed
278
                if remote[0] == socket.AF_INET6:
279
280
281
282
                    return Netstat.get_socket_owner_ipv6(local[4][0],
                                                         local[4][1],
                                                         remote[4][0],
                                                         remote[4][1])
Gero Müller's avatar
Gero Müller committed
283
        return None
Gero Müller's avatar
Gero Müller committed
284
285


286
class AjaxException(Exception):
287
    """
Marcel Rieger's avatar
Marcel Rieger committed
288
    AjaxException that is handled by the ajax tool and that can be raised in controller methods.
289
    *message* is the error message to show. *code* should be an integer that represents a specific
Marcel Rieger's avatar
Marcel Rieger committed
290
291
292
    type of exception. If *code* is *None* and *message* is an integer representing a http status
    code, the error message is set to the standard error message for that http error. If *alert* is
    *True*, the message is shown in a dialog in the GUI.
293
    """
294

295
296
297
298
299
300
301
302
303
304
305
    def __init__(self, message, code=None, alert=True):
        """
        __init__(message, code=500, alert=True)
        """
        if code is None:
            if isinstance(message, int) and message in response_codes:
                code    = message
                message = response_codes[code][0]
            else:
                code = 500

306
        super(AjaxException, self).__init__(message)
307
308
309

        self.code  = code
        self.alert = alert
310
311


312
313
# the bus
bus = None
Marcel Rieger's avatar
Marcel Rieger committed
314
315
316
317


def send_mail(addr, subject="", content="", sender_addr=None, smtp_host=None,
              smtp_port=None):
Gero Müller's avatar
Gero Müller committed
318
319
320
321
322
323
324
325
326
327
328
329
330
331
    """Send an email.

    All arguments should be Unicode strings (plain ASCII works as well).

    Only the real name part of sender and recipient addresses may contain
    non-ASCII characters.

    The email will be properly MIME encoded and delivered though SMTP to
    localhost port 25.  This is easy to change if you want something different.

    The charset of the email will be the first one out of US-ASCII, ISO-8859-1
    and UTF-8 that can represent all the characters occurring in the email.
    """

Gero Müller's avatar
Gero Müller committed
332
333
334
    if smtp_host is None:
        smtp_host = config("mail", "smtp.host", "localhost")
    if smtp_port is None:
Gero Müller's avatar
Gero Müller committed
335
        smtp_port = config("mail", "smtp.port", 0)
Gero Müller's avatar
Gero Müller committed
336
337
338
    if sender_addr is None:
        sender_addr = config("mail", "sender_address", "noreply@example.org")

Gero Müller's avatar
Gero Müller committed
339
340
341
    # Header class is smart enough to try US-ASCII, then the charset we
    # provide, then fall back to UTF-8.
    header_charset = 'ISO-8859-1'
Marcel Rieger's avatar
Marcel Rieger committed
342

Gero Müller's avatar
Gero Müller committed
343
344
345
346
347
348
349
350
    # We must choose the body charset manually
    for body_charset in 'US-ASCII', 'ISO-8859-1', 'UTF-8':
        try:
            content.encode(body_charset)
        except UnicodeError:
            pass
        else:
            break
Marcel Rieger's avatar
Marcel Rieger committed
351

Gero Müller's avatar
Gero Müller committed
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
    # Split real name (which is optional) and email address parts
    sender_name, sender_addr = parseaddr(sender_addr)
    recipient_name, recipient_addr = parseaddr(addr)

    # We must always pass Unicode strings to Header, otherwise it will
    # use RFC 2047 encoding even on plain ASCII strings.
    sender_name = str(Header(unicode(sender_name), header_charset))
    recipient_name = str(Header(unicode(recipient_name), header_charset))

    # Make sure email addresses do not contain non-ASCII characters
    sender_addr = sender_addr.encode('ascii')
    recipient_addr = recipient_addr.encode('ascii')

    # Create the message ('plain' stands for Content-Type: text/plain)
    msg = MIMEText(content.encode(body_charset), 'plain', body_charset)
    msg['From'] = formataddr((sender_name, sender_addr))
    msg['To'] = formataddr((recipient_name, recipient_addr))
    msg['Subject'] = Header(unicode(subject), header_charset)

    # Send the message via SMTP to localhost:25
Gero Müller's avatar
Gero Müller committed
372
    smtp = SMTP(smtp_host, smtp_port)
Gero Müller's avatar
Gero Müller committed
373
374
    smtp.sendmail(sender_addr, addr, msg.as_string())
    smtp.quit()
Gero Müller's avatar
Gero Müller committed
375
376
377
378
379
380
381
382
383
384


def thread_stacktraces():
    frames = sys._current_frames()
    data = {}
    for thread_id, frame in frames.iteritems():
        stacktrace = StringIO()
        traceback.print_stack(frame, file=stacktrace)
        data[thread_id] = stacktrace.getvalue()
    return data
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418


def dump_thread_status(f=None):
    # open the file
    if f == None:
        f = config('web', 'status_filename', None)

    if isinstance(f, (str, unicode)):
        f = f % {'pid': os.getpid()}
        f = open(f, 'a')

    if not hasattr(f, 'write'):
        return

    # header
    f.write("""
    <html>
    <head>
        <title>CherryPy Status</title>
    </head>
    <body>
    <h1>CherryPy Status</h1>
    """)

    # worker table
    f.write("""
    <table>
      <tr>
        <th>Thread ID</th>
        <th>Idle Time</th>
        <th>Last Request Time</th>
        <th>URL</th>
      </tr>
    """)
Gero Müller's avatar
Gero Müller committed
419
420
421
    threads = copy.deepcopy(cherrypy.tools.status.seen_threads.values())
    threads.sort(key=lambda thread: (thread.idle_time(), thread.last_req_time()))
    for thread in threads:
422
423
424
425
426
427
        f.write("""
        <tr>
          <th><a href=\"#%s\">%s</a></th>
          <td>%.4f</td>
          <td>%.4f</td>
          <td>%s</td>
Gero Müller's avatar
Gero Müller committed
428
429
         </tr>""" % (thread.id, thread.id, thread.idle_time(),
                     thread.last_req_time(), thread.url))
430
431
432
433
434
435
436
437
438
439
440
441
442
    f.write("</table>")

    # threads
    for thread_id, st in thread_stacktraces().items():
        f.write("""
        <h3><a id=\"%s\">%s</a></h3>
        <pre>\n%s</pre>""" % (thread_id, thread_id, st))

    f.write("""
    </body>
    </html>
    """)

Gero Müller's avatar
Gero Müller committed
443
444
    if hasattr(f, 'flush'):
        f.flush()
Gero Müller's avatar
Gero Müller committed
445

446
447
448
449
450
451
452

def dump_thread_status_on_signal(signal, stack):
    f = os.path.join(tempfile.gettempdir(),
                     "cherrypy-status-%d.html" % os.getpid())
    dump_thread_status(f)


Gero Müller's avatar
Gero Müller committed
453
454
def setup_thread_dump():
    signal.signal(signal.SIGUSR2, dump_thread_status_on_signal)