jobs will be killed after some time

requirements.txt

@@ -14,4 +14,4 @@ pyexcel-xls==0.5.0
 pyexcel==0.5.3
 pycurl==7.43.0
 django-stronghold==0.2.9
-
+timeout-decorator==0.4.0

smash/web/models/constants.py

@@ -81,6 +81,9 @@ WEEKDAY_CHOICES = (
 REDCAP_TOKEN_CONFIGURATION_TYPE = "REDCAP_TOKEN_CONFIGURATION_TYPE"
 REDCAP_BASE_URL_CONFIGURATION_TYPE = "REDCAP_BASE_URL_CONFIGURATION_TYPE"
 
+# timeout job after 15 minutes
+CRON_JOB_TIMEOUT = 15 * 60 * 1000
+
 COUNTRY_OTHER_ID = 1
 COUNTRY_AFGHANISTAN_ID = 2
 

smash/web/redcap_connector.py

@@ -5,11 +5,12 @@ import logging
 import pycurl
 
 import certifi
+import timeout_decorator
 from django_cron import CronJobBase, Schedule
 
 from web.models import ConfigurationItem, StudySubject, Language
 from web.models.constants import REDCAP_TOKEN_CONFIGURATION_TYPE, \
-    REDCAP_BASE_URL_CONFIGURATION_TYPE
+    REDCAP_BASE_URL_CONFIGURATION_TYPE, CRON_JOB_TIMEOUT
 from web.models.inconsistent_subject import InconsistentField, InconsistentSubject
 from web.models.missing_subject import MissingSubject
 
@@ -322,6 +323,7 @@ class RedCapRefreshJob(CronJobBase):
     schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
     code = 'web.red_cap_hourly_refresh'  # a unique code
 
+    @timeout_decorator.timeout(CRON_JOB_TIMEOUT, use_signals=False)
     def do(self):
         connector = RedcapConnector()
         if connector.is_valid():

smash/web/smash_email.py

@@ -3,6 +3,7 @@
 import logging
 
 from django.conf import settings
+from django.core import mail
 from django.core.mail import EmailMessage
 
 logger = logging.getLogger(__name__)
@@ -26,5 +27,7 @@ class EmailSender(object):
             cc=cc_recipients
         )
         message.content_subtype = "html"
+        print "mails sent: " + str(len(mail.outbox))
         message.send()
+        print "mails sent: " + str(len(mail.outbox))
         logger.info('Email sent. Subject: ' + subject + "; Recipients: " + recipients)

smash/web/tests/view/test_KitRequestEmailSendJob.py

@@ -37,5 +37,6 @@ class KitRequestEmailSendJobTests(LoggedInTestCase):
         status = job.do()
 
         self.assertEqual("mail sent", status)
+        print len(mail.outbox)
         self.assertEqual(1, len(mail.outbox))
         self.assertEqual(workers_count, Worker.objects.all().count())

smash/web/views/kit.py

@@ -6,6 +6,7 @@ import platform
 import time
 
 import pytz
+import timeout_decorator
 from django.contrib import messages
 from django.utils.dateparse import parse_datetime
 from django_cron import CronJobBase, Schedule
@@ -14,7 +15,7 @@ from django_cron.models import CronJobLog
 from notifications import get_filter_locations, get_today_midnight_date
 from web.models import ConfigurationItem, Language, Worker
 from web.models.constants import KIT_EMAIL_HOUR_CONFIGURATION_TYPE, \
-    KIT_EMAIL_DAY_OF_WEEK_CONFIGURATION_TYPE
+    KIT_EMAIL_DAY_OF_WEEK_CONFIGURATION_TYPE, CRON_JOB_TIMEOUT
 from web.models.constants import KIT_RECIPIENT_EMAIL_CONFIGURATION_TYPE
 from . import wrap_response
 from ..forms import KitRequestForm
@@ -140,25 +141,37 @@ class KitRequestEmailSendJob(CronJobBase):
     schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
     code = 'web.kit_request_weekly_email'  # a unique code
 
+    @timeout_decorator.timeout(CRON_JOB_TIMEOUT, use_signals=False)
     def do(self):
+        print "X"
         now = datetime.datetime.utcnow()
+        print "X4"
         hour = int(ConfigurationItem.objects.get(
             type=KIT_EMAIL_HOUR_CONFIGURATION_TYPE).value.split(":")[0])
+        print "X3"
         minute = int(ConfigurationItem.objects.get(
             type=KIT_EMAIL_HOUR_CONFIGURATION_TYPE).value.split(":")[1])
+        print "2"
         # check if we sent email this day already
         date = now.replace(hour=hour, minute=minute)
+        print "1"
         # TODO it's a hack assuming that we are in CEST
         date = pytz.utc.localize(date - datetime.timedelta(minutes=122))
+        print "a"
         jobs = CronJobLog.objects.filter(code=KitRequestEmailSendJob.code, message="mail sent", start_time__gte=date)
+        print "Y"
 
         if jobs.count() == 0:
+            print "Ya"
             if pytz.utc.localize(datetime.datetime.utcnow()) > date:
+                print "Ys"
                 if self.match_day_of_week():
+                    print "Yd"
                     worker = Worker.objects.create()
                     data = get_kit_requests(worker)
                     send_mail(data)
                     worker.delete()
+                    print "Ye"
                     return "mail sent"
                 else:
                     return "day of week doesn't match"

smash/web/views/voucher.py

 # coding=utf-8
 import logging
 
+import timeout_decorator
 from django.contrib.messages.views import SuccessMessageMixin
 from django.urls import reverse_lazy
 from django.views.generic import CreateView
@@ -11,7 +12,7 @@ from django_cron import CronJobBase, Schedule
 from web.views.notifications import get_today_midnight_date
 from web.forms import VoucherForm
 from web.models import Voucher, StudySubject, MailTemplate
-from web.models.constants import GLOBAL_STUDY_ID, VOUCHER_STATUS_NEW, VOUCHER_STATUS_EXPIRED
+from web.models.constants import GLOBAL_STUDY_ID, VOUCHER_STATUS_NEW, VOUCHER_STATUS_EXPIRED, CRON_JOB_TIMEOUT
 from . import WrappedView
 
 logger = logging.getLogger(__name__)
@@ -80,6 +81,7 @@ class ExpireVouchersJob(CronJobBase):
     code = 'web.voucher_expiry_job'  # a unique code
 
     # noinspection PyMethodMayBeStatic
+    @timeout_decorator.timeout(CRON_JOB_TIMEOUT, use_signals=False)
     def do(self):
         due_date = get_today_midnight_date()
         vouchers = Voucher.objects.filter(status=VOUCHER_STATUS_NEW, expiry_date__lte=due_date)