updated script to support new sheets in excel and extra information

smash/import_file.py

@@ -25,7 +25,7 @@ from web.utils import is_valid_social_security_number
 from web.models import VoucherType, Voucher, Country, AppointmentTypeLink, AppointmentType, Study, Worker, Language, Subject, WorkerStudyRole, StudySubject, Location, FlyingTeam, Visit, Appointment, AppointmentType
 
 from web.models.worker_study_role import ROLE_CHOICES_TECHNICIAN, WORKER_STAFF, ROLE_CHOICES_SECRETARY, ROLE_CHOICES_HEALTH_PARTNER, \
-    WORKER_HEALTH_PARTNER, ROLE_CHOICES_VOUCHER_PARTNER, WORKER_VOUCHER_PARTNER
+    WORKER_HEALTH_PARTNER, ROLE_CHOICES_DOCTOR, ROLE_CHOICES_VOUCHER_PARTNER, ROLE_CHOICES, ROLE_CHOICES_NURSE, ROLE_CHOICES_PSYCHOLOGIST, ROLE_CHOICES_PROJECT_MANAGER
 
 DEFAULT_LOCATION = 'CHL'
 DEFAULT_LOCATION_PREFIX = 'P'
@@ -149,7 +149,13 @@ def parse_boolean(boolean_Y_N):
 
 
 def parse_column_date_of_birth(date):
-    return datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y-%m-%d')
+    try:
+        return datetime.datetime.strptime(date, '%d.%m.%Y %H:%M').strftime('%Y-%m-%d')
+    except ValueError:
+        try:
+            return datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y-%m-%d')
+        except:
+            return None
 
 # gender
 gender_table = {'m': SEX_CHOICES_MALE, 'f': SEX_CHOICES_FEMALE}
@@ -179,6 +185,18 @@ def parse_column_ss_number(ss):
         logging.debug('Invalid SS number: (Length not valid) |{}|'.format(ss))
     return ss
 
+def parse_role(role_name):
+    d = {
+        'NURSE': ROLE_CHOICES_NURSE,
+        'PSYCHOLOGIST': ROLE_CHOICES_PSYCHOLOGIST,
+        'SECRETARY': ROLE_CHOICES_SECRETARY,
+        'PROJECT MANAGER': ROLE_CHOICES_PROJECT_MANAGER,
+        'DOCTOR': ROLE_CHOICES_DOCTOR,
+        'HEALTH_PARTNER': ROLE_CHOICES_HEALTH_PARTNER,
+        'VOUCHER_PARTNER': ROLE_CHOICES_VOUCHER_PARTNER
+    }
+    return d[role_name] #crash if no valid role_name is given
+
 # Language
 
 
@@ -188,6 +206,7 @@ language_table = {
     'IT': 'Italian',
     'F': 'French',
     'D': 'German',
+    'G': 'German',
     'E': 'English',
     'P': 'Portuguese',
     'A': 'Arabic',
@@ -278,7 +297,7 @@ converters = {
 # add voucher for subject
 voucher_partners = {}
 voucher_partners['ZIT'] = 'Zitha'
-def add_subject_vouchers(voucher_reference, referral, voucher_types):
+def add_subject_vouchers(voucher_reference, referral, voucher_types, subject_nd_number):
     nd_number, date, voucher_partner, voucher_type, num = voucher_reference.split('-')
     nd_number = nd_number.upper().replace('ND', 'PDP')
     issue_date = datetime.datetime.strptime(date, '%Y%m%d')
@@ -297,7 +316,9 @@ def add_subject_vouchers(voucher_reference, referral, voucher_types):
 
     vt = VoucherType.objects.get(code=voucher_type)
     
-    study_subject = StudySubject.objects.get(nd_number=nd_number)
+    if nd_number != subject_nd_number:
+        logging.warn('voucher reference nd_number is not the same! {} != {}'.format(nd_number, subject_nd_number))
+    study_subject = StudySubject.objects.get(nd_number=subject_nd_number)
 
     voucher, created = Voucher.objects.update_or_create(number=voucher_reference, issue_date=issue_date, 
         expiry_date=expiry_date, voucher_type=vt, study_subject=study_subject, 
@@ -305,8 +326,21 @@ def add_subject_vouchers(voucher_reference, referral, voucher_types):
     logging.warn('New Voucher added: {}'.format(voucher_reference))
     return voucher
 
-
 # create voucher types
+voucher_type_codes = set()
+def create_voucher_code(description):
+    code = re.sub(r'\([^)]*\)', '', description.strip())
+    code = re.sub(r'[aeiou]+', '', code.lower()).strip()
+    code = ''.join([e.capitalize() for e in code.split(' ')[:2]])
+    while code in voucher_type_codes:
+        if re.match(r'.*_\d+$', code) is None:
+            code = code + '_2'
+        else:
+            spt = code.split('_')
+            num = int(spt[-1])+1
+            code = ''.join(spt[:len(spt)]) + str(num)
+    return code
+
 def create_voucher_types(voucher_types_dict, study):
     voucher_types = {}
     for name, code in voucher_types_dict.items():
@@ -325,21 +359,11 @@ def create_appointment_types(assessments):
         appointmentTypes.append(appointmentType)
     return appointmentTypes
 
-
-def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_worker):
-    # Languages
-    if len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
-        logging.warn('No Languages available')
-    elif len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) > 0:
-        row['LANGUAGES'] = row['PREFERED WRITEN LANGUAGE']
-    elif len(row['LANGUAGES']) > 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
-        row['PREFERED WRITEN LANGUAGE'] = row['LANGUAGES']
-
+def create_languages(languages_cell):
     languages = []
-    for language in row['LANGUAGES']:
+    for language in languages_cell:
         lang, created = Language.objects.get_or_create(
             name=language, locale=locale_table.get(language,(None, None))[0])
-        languages.append(lang)
         if created:
             logging.warn('New Language added: {}'.format(language))
 
@@ -355,13 +379,25 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
             lang.image = basename
             lang.save()
 
-    for language in row['PREFERED WRITEN LANGUAGE'][:1]:
-        pref_lang, created = Language.objects.get_or_create(name=language
-            ,locale=locale_table.get(language,(None, None))[0])
-        if created:
-            logging.warn(
-                'New Language (from Prefered) added: {}'.format(language))
-        pref_lang.save()
+        languages.append(lang)
+
+    return languages
+
+def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_worker):
+    # Languages
+    if len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
+        logging.warn('No Languages available')
+    elif len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) > 0:
+        row['LANGUAGES'] = row['PREFERED WRITEN LANGUAGE']
+    elif len(row['LANGUAGES']) > 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
+        row['PREFERED WRITEN LANGUAGE'] = row['LANGUAGES']
+
+    languages = create_languages(row['LANGUAGES'])
+    pref_lang = create_languages(row['PREFERED WRITEN LANGUAGE'][:1])
+    if len(pref_lang) > 0:
+        pref_lang = pref_lang[0]
+    elif len(languages) > 0:
+        pref_lang = languages[0]
 
     # Country
     country = row['COUNTRY']
@@ -399,15 +435,20 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
 
     # Health Partner
     # create health partner (Referral)
-    health_partner, created = Worker.objects.get_or_create(name=row['REFERRAL'])
-    health_partner.roles.update(role=ROLE_CHOICES_HEALTH_PARTNER)
-    # create workerStudyRole
-    workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(
-        worker=health_partner, study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_HEALTH_PARTNER)
-    health_partner.save()
-    if created:
-        logging.warn('New Health Partner added: {}'.format(row['REFERRAL']))
+    health_partner = None
+    if row['REFERRAL'].strip() != '':
+        logging.warn('Trying to get or create Worker: {}'.format(row['REFERRAL']))
+        health_partner, created = Worker.objects.get_or_create(name=row['REFERRAL'])
+        health_partner.roles.update(role=ROLE_CHOICES_HEALTH_PARTNER)
+        # create workerStudyRole
+        workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(
+            worker=health_partner, study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_HEALTH_PARTNER)
+        health_partner.save()
+        if created:
+            logging.warn('New Health Partner added: {}'.format(row['REFERRAL']))
 
+    if row['SS NUMBER'] is None:
+        row['SS NUMBER'] = ''
     subject, created = Subject.objects.get_or_create(social_security_number=row['SS NUMBER'],
                                                      first_name=row['FIRST NAME'],
                                                      last_name=row['LAST NAME'],
@@ -466,7 +507,7 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
     #VOUCHERS
     voucher_references = row['VOUCHER REFERENCE']
     for voucher_reference in voucher_references:
-        voucher = add_subject_vouchers(voucher_reference, health_partner, voucher_types)
+        voucher = add_subject_vouchers(voucher_reference, health_partner, voucher_types, nd_number)
 
     # Visits
     # Consider all visits as part of the same visit with multiple appointments
@@ -478,8 +519,14 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
     sum(Ans, []) flattens the resulting list from the map since each findall returns a list
     map to convert string to datetime
     '''
-    visit_dates = map(lambda x: datetime.datetime.strptime(
-        x, '%d.%m.%Y'), sum(map(date_regex.findall, row[visit_columns].values), []))
+    def parse_visit_date(date_string):
+        try:
+            dtime = datetime.datetime.strptime(date_string, '%d.%m.%Y %H:%M')
+        except ValueError:
+            dtime = datetime.datetime.strptime(date_string, '%d.%m.%Y')
+        return dtime
+
+    visit_dates = map(lambda x: parse_visit_date(x), sum(map(date_regex.findall, row[visit_columns].values), []))
 
     # get first and last elements of the sorted element
     datetime_begin, datetime_end = itemgetter(*[0, -1])(sorted(visit_dates))
@@ -502,8 +549,9 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
     '''
     starting_hour = 9
     for visit_date in set(visit_dates):
-        datetime_when = visit_date.replace(hour=starting_hour, minute=0, second=0, microsecond=0)
-        starting_hour+=1
+        if visit_date.hour == 0:
+            datetime_when = visit_date.replace(hour=starting_hour, minute=0, second=0, microsecond=0)
+            starting_hour+=1
 
         #  get the indices of each occurrence of the date and use them to get
         # the appointment types
@@ -528,34 +576,37 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
             app_type_link.save()
 
 def createWorker(password, email='', username='admin', first_name='LCSB', last_name='Admin',
-    specialization='Technician', unit='LCSB'):
-
-        # create user
-        defaults = {'email': email, 'password': password}
-        user, _ = User.objects.update_or_create(username=username, defaults=defaults)
-        user.set_password(password)
-        user.is_superuser = True
-        user.is_staff = True
-        user.is_admin = True
-        user.save()
+    specialization='Technician', unit='LCSB', languages=[], role=ROLE_CHOICES_TECHNICIAN, 
+    name='', comment=None):
+        user = None
+        if username is not None: 
+            # create user
+            defaults = {'email': email, 'password': password}
+            user, _ = User.objects.update_or_create(username=username, defaults=defaults)
+            user.set_password(password)
+            user.is_superuser = True
+            user.is_staff = True
+            user.is_admin = True
+            user.save()
 
         # create worker
         defaults = {'first_name': first_name, 'last_name': last_name,
                     'email': email, 'unit': unit, 'specialization': specialization
-                    , 'user': user}
+                    , 'user': user, 'name': name, 'comment': comment}
+    
         worker, _ = Worker.objects.update_or_create(first_name=first_name,
                                                     last_name=last_name, defaults=defaults)
-
         
         locations = Location.objects.all()
         worker.locations.set(locations)
-        languages = Language.objects.all()
+        if len(languages) == 0:
+            languages = Language.objects.all()
         worker.languages.set(languages)
         worker.save()
 
         # create workerStudyRole
         workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(worker=worker,
-                                                                      study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_TECHNICIAN)
+                                                                      study_id=GLOBAL_STUDY_ID, role=role)
         workerStudyRole.save()
         logging.info('SuperUser and Worker {} created'.format(username))
         return worker
@@ -570,16 +621,66 @@ if __name__ == '__main__':
         logging.warn('Please, execute the program with a valid file path.')
         sys.exit(1)
 
+    admin_password = None
+    if len(sys.argv) >= 3:
+        admin_password = sys.argv[2]
+
     #create worker and super user
-    pass1 = ''
-    pass2 = None
-    while pass1 != pass2:
-        pass1 = getpass.getpass('Please type a password for the Admin user: ')
-        pass2 = getpass.getpass('Please type your password again: ')
-        if pass1 != pass2:
-            print 'Password mismatch, please try again'
+    if admin_password is None:
+        pass1 = ''
+        pass2 = None
+        while pass1 != pass2:
+            pass1 = getpass.getpass('Please type a password for the Admin user: ')
+            pass2 = getpass.getpass('Please type your password again: ')
+            if pass1 != pass2:
+                print 'Password mismatch, please try again'
+    else:
+        pass1 = admin_password
     lcsb_worker = createWorker(pass1)
 
+    #STUDY
+    study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+    #enable vouchers
+    study.columns.voucher_types = True
+    study.columns.vouchers = True
+    study.nd_number_study_subject_regex = r'^PDP\d{4}$'
+    study.columns.save()
+    study.save()
+
+    #READ WORKERS SHEET
+    df = pd.read_excel(file, dtype=object, sheet_name='Workers')
+    #convert column name to upper case
+    df.columns = [c.upper() for c in df.columns]
+    #convert languages
+    df['LANGUAGES'] = df['LANGUAGES'].apply(apply_column_languages)
+    df['ROLE'] = df['ROLE'].apply(parse_role)
+    for index, row in df.iterrows():
+        languages = create_languages(row['LANGUAGES'])
+        createWorker(row['PASSWORD'], email=row['MAIL'], username=row['USERNAME'], first_name=row['FIRST NAME'], last_name=row['LAST NAME'],
+    specialization=row['ROLE'], role=row['ROLE'], unit='CHL', languages=languages)
+        
+    #READ VOUCHER PARTNER SHEET
+    df = pd.read_excel(file, dtype=object, sheet_name='Voucher Partners')
+    #convert column name to upper case
+    df.columns = [c.upper() for c in df.columns]
+    df['ROLE'] = df['ROLE'].apply(parse_role)
+    for index, row in df.iterrows():
+        worker = createWorker(None, email='', username=None, first_name='', last_name='', specialization=row['ROLE'], 
+            role=row['ROLE'], unit='CHL', languages=[], name=row['NAME'], comment=row.get('COMMENT', ''))
+        if type(row['OFFER']) != float:
+            offers = row['OFFER'].split(';')
+            voucher_types = []
+            for offer in offers:
+                code = create_voucher_code(offer)
+                voucher_type_codes.add(code)
+                voucher_type, created = VoucherType.objects.update_or_create(description=offer.title(), study=study, code=code)
+                if created:
+                    logging.warn(u'Voucher type created: {} ({})'.format(offer, code))
+                voucher_types.append(voucher_type)
+            worker.voucher_types.set(voucher_types)
+            worker.save()
+
+    #READ FIRST SHEET
     df = pd.read_excel(file, dtype=object)
     df = df.fillna('').astype(unicode)
     df.columns = [c.upper() for c in df.columns]
@@ -590,14 +691,6 @@ if __name__ == '__main__':
     # get visits columns
     regex = re.compile(r'\(V\d\)')
     # 
-    study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
-    #enable vouchers
-    study.columns.voucher_types = True
-    study.columns.vouchers = True
-    study.nd_number_study_subject_regex = r'^PDP\d{4}$'
-    study.columns.save()
-    study.save()
-    #
     visit_columns = filter(regex.search, df.columns)
 
     assessments = OrderedDict([('Cognitive Test', 180), ('Risk Factor', 120),