>From a16ada793a0b6750d01e5fd05172ba6f9f6c0cb0 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 29 Sep 2015 18:36:20 +0300 Subject: [PATCH 1/4] Add whole thread[as mbox] link to message view diff --git a/django/archives/mailarchives/templates/message.html b/django/archives/mailarchives/templates/message.html index 1d09e24..d0eeebf 100644 --- a/django/archives/mailarchives/templates/message.html +++ b/django/archives/mailarchives/templates/message.html @@ -45,7 +45,7 @@ $(function(){ Message-ID: - {{msg.messageid}} (view raw or whole thread) + {{msg.messageid}} (view raw or whole thread [as mbox]) Thread: diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py index e011827..bfe0f26 100644 --- a/django/archives/mailarchives/views.py +++ b/django/archives/mailarchives/views.py @@ -255,6 +255,19 @@ SELECT id,_from,subject,date,messageid,has_attachment,parentid,datepath FROM t O for id,_from,subject,date,messageid,has_attachment,parentid,parentpath in curs.fetchall(): yield {'id':id, 'mailfrom':_from, 'subject': subject, 'date': date, 'printdate': date.strftime("%Y-%m-%d %H:%M:%S"), 'messageid': messageid, 'hasattachment': has_attachment, 'parentid': parentid, 'indent': " " * len(parentpath)} +def _retrieve_thread_raw_messages(threadid): + # Yeah, this is *way* too complicated for the django ORM Too + curs = connection.cursor() + curs.execute("""WITH RECURSIVE t(id, date, rawtxt) AS( + SELECT id, date, rawtxt FROM messages m WHERE m.threadid=%(threadid)s AND parentid IS NULL AND hiddenstatus IS NULL + UNION ALL + SELECT m.id, m.date, m.rawtxt FROM messages m INNER JOIN t ON t.id=m.parentid WHERE m.threadid=%(threadid)s AND hiddenstatus IS NULL +) +SELECT id, date, rawtxt FROM t; +""", {'threadid': threadid}) + + for id, date,rawtxt in curs.fetchall(): + yield {'id':id, 'date': date, 'rawtext': rawtxt} def _get_nextprevious(listmap, dt): curs = connection.cursor() @@ -390,6 +403,46 @@ def message_raw(request, msgid): response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"' return response +@nocache +def message_thread_raw(request, msgid): + if 'HTTP_AUTHORIZATION' in request.META: + auth = request.META['HTTP_AUTHORIZATION'].split() + if len(auth) != 2: + return HttpResponseForbidden("Invalid authentication") + if auth[0].lower() == "basic": + user, pwd = base64.b64decode(auth[1]).split(':') + if user == 'archives' and pwd == 'antispam': + try: + m = Message.objects.get(messageid=msgid) + except Message.DoesNotExist: + raise Http404('Message does not exist') + + msgs = list(_retrieve_thread_raw_messages(m.threadid)) + newest = calendar.timegm(max(msgs, key=lambda x: x['date'])['date'].utctimetuple()) + if request.META.has_key('HTTP_IF_MODIFIED_SINCE') and not settings.DEBUG: + ims = parse_http_date_safe(request.META.get("HTTP_IF_MODIFIED_SINCE")) + if ims >= newest: + return HttpResponseNotModified() + + payload, msgs = bytes(msgs[0]['rawtext']), msgs[1:] + while msgs: + msg,msgs = msgs[0], msgs[1:] + payload += b"\n"+bytes(msg['rawtext']) + + r = HttpResponse(payload, content_type='text/plain') + r['X-pgthread'] = ":%s:" % m.threadid + return r + + # Invalid password falls through + # Other authentication types fall through + + # Require authentication + response = HttpResponse() + response.status_code = 401 + response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"' + return response + + def testview(request, seqid): m = Message.objects.get(pk=seqid) try: diff --git a/django/archives/urls.py b/django/archives/urls.py index 16fc718..5ca545c 100644 --- a/django/archives/urls.py +++ b/django/archives/urls.py @@ -27,6 +27,7 @@ urlpatterns = patterns('', (r'^message-id/([^/]+)$', 'archives.mailarchives.views.message'), (r'^message-id/flat/([^/]+)$', 'archives.mailarchives.views.message_flat'), (r'^message-id/raw/([^/]+)$', 'archives.mailarchives.views.message_raw'), + (r'^message-id/thread_raw/([^/]+)$', 'archives.mailarchives.views.message_thread_raw'), # Search (r'^archives-search/', 'archives.mailarchives.views.search'), -- 2.4.3 >From b998a628c8b43a851ea96762ee694cc4803be645 Mon Sep 17 00:00:00 2001 From: Amir Date: Wed, 30 Sep 2015 02:49:41 +0300 Subject: [PATCH 2/4] Quote 'archives'/'antispam' in Basic Auth message diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py index bfe0f26..45da93d 100644 --- a/django/archives/mailarchives/views.py +++ b/django/archives/mailarchives/views.py @@ -400,7 +400,7 @@ def message_raw(request, msgid): # Require authentication response = HttpResponse() response.status_code = 401 - response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"' + response['WWW-Authenticate'] = "Basic realm=\"Please authenticate with user 'archives' and password 'antispam'\"" return response @nocache @@ -439,7 +439,7 @@ def message_thread_raw(request, msgid): # Require authentication response = HttpResponse() response.status_code = 401 - response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"' + response['WWW-Authenticate'] = "Basic realm=\"Please authenticate with user 'archives' and password 'antispam'\"" return response -- 2.4.3 >From 3bd67deb2172e4c57b84c2471e8c3f39fb927a40 Mon Sep 17 00:00:00 2001 From: Amir Date: Wed, 30 Sep 2015 03:28:08 +0300 Subject: [PATCH 3/4] Add settings.THREAD_MBOX_MAX_SIZE size limit on thread mbox response size diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py index 45da93d..78c8d8f 100644 --- a/django/archives/mailarchives/views.py +++ b/django/archives/mailarchives/views.py @@ -417,18 +417,21 @@ def message_thread_raw(request, msgid): except Message.DoesNotExist: raise Http404('Message does not exist') - msgs = list(_retrieve_thread_raw_messages(m.threadid)) - newest = calendar.timegm(max(msgs, key=lambda x: x['date'])['date'].utctimetuple()) + payload, newest = b"", None + for msg in _retrieve_thread_raw_messages(m.threadid): + if newest is None: + newest = msg['date'] + newest = max(newest , msg['date'] ) + payload += b"\n"+bytes(msg['rawtext']) + if len(payload) > settings.THREAD_MBOX_MAX_SIZE: + return HttpResponseForbidden('Thread exceeded size limits.') + + newest = calendar.timegm(newest.utctimetuple()) if request.META.has_key('HTTP_IF_MODIFIED_SINCE') and not settings.DEBUG: ims = parse_http_date_safe(request.META.get("HTTP_IF_MODIFIED_SINCE")) if ims >= newest: return HttpResponseNotModified() - payload, msgs = bytes(msgs[0]['rawtext']), msgs[1:] - while msgs: - msg,msgs = msgs[0], msgs[1:] - payload += b"\n"+bytes(msg['rawtext']) - r = HttpResponse(payload, content_type='text/plain') r['X-pgthread'] = ":%s:" % m.threadid return r diff --git a/django/archives/settings.py b/django/archives/settings.py index 0a01c3b..57c06a8 100644 --- a/django/archives/settings.py +++ b/django/archives/settings.py @@ -155,6 +155,13 @@ MBOX_ARCHIVES_ROOT="/dev/null" SEARCH_CLIENTS = ('127.0.0.1',) API_CLIENTS = ('127.0.0.1',) +# Hard limits applicable to on-the-fly mbox generation +# see views.message_thread_raw. + +# Max size of response, in bytes +# Enforced only *after* the databse fetches the data. +THREAD_MBOX_MAX_SIZE = 1000000 + try: from settings_local import * except ImportError: -- 2.4.3 >From 49aceb5a8b8dc13514cc0f20f64547de37a6801c Mon Sep 17 00:00:00 2001 From: Amir Date: Wed, 30 Sep 2015 04:17:06 +0300 Subject: [PATCH 4/4] Add settings.THREAD_MBOX_DISABLED emergency lever diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py index 78c8d8f..906d279 100644 --- a/django/archives/mailarchives/views.py +++ b/django/archives/mailarchives/views.py @@ -405,6 +405,9 @@ def message_raw(request, msgid): @nocache def message_thread_raw(request, msgid): + if settings.THREAD_MBOX_DISABLED: + return HttpResponseForbidden("Downloading entire threads in mbox format is currently disabled.") + if 'HTTP_AUTHORIZATION' in request.META: auth = request.META['HTTP_AUTHORIZATION'].split() if len(auth) != 2: diff --git a/django/archives/settings.py b/django/archives/settings.py index 57c06a8..fc0fd6f 100644 --- a/django/archives/settings.py +++ b/django/archives/settings.py @@ -162,6 +162,10 @@ API_CLIENTS = ('127.0.0.1',) # Enforced only *after* the databse fetches the data. THREAD_MBOX_MAX_SIZE = 1000000 +# "The Ops Eject button": set this to true to turn all +# requests into cheap 403's +THREAD_MBOX_DISABLED = False + try: from settings_local import * except ImportError: -- 2.4.3