diff --git a/smash/import_file.py b/smash/import_file.py
index 6eff049dc433c8aa9c7e53ffa7934e4acb7061e1..c46e183646632e7a15627ae327a33f0b0ea63239 100644
--- a/smash/import_file.py
+++ b/smash/import_file.py
@@ -25,7 +25,7 @@ from web.utils import is_valid_social_security_number
from web.models import VoucherType, Voucher, Country, AppointmentTypeLink, AppointmentType, Study, Worker, Language, Subject, WorkerStudyRole, StudySubject, Location, FlyingTeam, Visit, Appointment, AppointmentType
from web.models.worker_study_role import ROLE_CHOICES_TECHNICIAN, WORKER_STAFF, ROLE_CHOICES_SECRETARY, ROLE_CHOICES_HEALTH_PARTNER, \
- WORKER_HEALTH_PARTNER, ROLE_CHOICES_VOUCHER_PARTNER, WORKER_VOUCHER_PARTNER
+ WORKER_HEALTH_PARTNER, ROLE_CHOICES_DOCTOR, ROLE_CHOICES_VOUCHER_PARTNER, ROLE_CHOICES, ROLE_CHOICES_NURSE, ROLE_CHOICES_PSYCHOLOGIST, ROLE_CHOICES_PROJECT_MANAGER
DEFAULT_LOCATION = 'CHL'
DEFAULT_LOCATION_PREFIX = 'P'
@@ -149,7 +149,13 @@ def parse_boolean(boolean_Y_N):
def parse_column_date_of_birth(date):
- return datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y-%m-%d')
+ try:
+ return datetime.datetime.strptime(date, '%d.%m.%Y %H:%M').strftime('%Y-%m-%d')
+ except ValueError:
+ try:
+ return datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y-%m-%d')
+ except:
+ return None
# gender
gender_table = {'m': SEX_CHOICES_MALE, 'f': SEX_CHOICES_FEMALE}
@@ -179,6 +185,18 @@ def parse_column_ss_number(ss):
logging.debug('Invalid SS number: (Length not valid) |{}|'.format(ss))
return ss
+def parse_role(role_name):
+ d = {
+ 'NURSE': ROLE_CHOICES_NURSE,
+ 'PSYCHOLOGIST': ROLE_CHOICES_PSYCHOLOGIST,
+ 'SECRETARY': ROLE_CHOICES_SECRETARY,
+ 'PROJECT MANAGER': ROLE_CHOICES_PROJECT_MANAGER,
+ 'DOCTOR': ROLE_CHOICES_DOCTOR,
+ 'HEALTH_PARTNER': ROLE_CHOICES_HEALTH_PARTNER,
+ 'VOUCHER_PARTNER': ROLE_CHOICES_VOUCHER_PARTNER
+ }
+ return d[role_name] #crash if no valid role_name is given
+
# Language
@@ -188,6 +206,7 @@ language_table = {
'IT': 'Italian',
'F': 'French',
'D': 'German',
+ 'G': 'German',
'E': 'English',
'P': 'Portuguese',
'A': 'Arabic',
@@ -278,7 +297,7 @@ converters = {
# add voucher for subject
voucher_partners = {}
voucher_partners['ZIT'] = 'Zitha'
-def add_subject_vouchers(voucher_reference, referral, voucher_types):
+def add_subject_vouchers(voucher_reference, referral, voucher_types, subject_nd_number):
nd_number, date, voucher_partner, voucher_type, num = voucher_reference.split('-')
nd_number = nd_number.upper().replace('ND', 'PDP')
issue_date = datetime.datetime.strptime(date, '%Y%m%d')
@@ -297,7 +316,9 @@ def add_subject_vouchers(voucher_reference, referral, voucher_types):
vt = VoucherType.objects.get(code=voucher_type)
- study_subject = StudySubject.objects.get(nd_number=nd_number)
+ if nd_number != subject_nd_number:
+ logging.warn('voucher reference nd_number is not the same! {} != {}'.format(nd_number, subject_nd_number))
+ study_subject = StudySubject.objects.get(nd_number=subject_nd_number)
voucher, created = Voucher.objects.update_or_create(number=voucher_reference, issue_date=issue_date,
expiry_date=expiry_date, voucher_type=vt, study_subject=study_subject,
@@ -305,8 +326,21 @@ def add_subject_vouchers(voucher_reference, referral, voucher_types):
logging.warn('New Voucher added: {}'.format(voucher_reference))
return voucher
-
# create voucher types
+voucher_type_codes = set()
+def create_voucher_code(description):
+ code = re.sub(r'\([^)]*\)', '', description.strip())
+ code = re.sub(r'[aeiou]+', '', code.lower()).strip()
+ code = ''.join([e.capitalize() for e in code.split(' ')[:2]])
+ while code in voucher_type_codes:
+ if re.match(r'.*_\d+$', code) is None:
+ code = code + '_2'
+ else:
+ spt = code.split('_')
+ num = int(spt[-1])+1
+ code = ''.join(spt[:len(spt)]) + str(num)
+ return code
+
def create_voucher_types(voucher_types_dict, study):
voucher_types = {}
for name, code in voucher_types_dict.items():
@@ -325,21 +359,11 @@ def create_appointment_types(assessments):
appointmentTypes.append(appointmentType)
return appointmentTypes
-
-def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_worker):
- # Languages
- if len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
- logging.warn('No Languages available')
- elif len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) > 0:
- row['LANGUAGES'] = row['PREFERED WRITEN LANGUAGE']
- elif len(row['LANGUAGES']) > 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
- row['PREFERED WRITEN LANGUAGE'] = row['LANGUAGES']
-
+def create_languages(languages_cell):
languages = []
- for language in row['LANGUAGES']:
+ for language in languages_cell:
lang, created = Language.objects.get_or_create(
name=language, locale=locale_table.get(language,(None, None))[0])
- languages.append(lang)
if created:
logging.warn('New Language added: {}'.format(language))
@@ -355,13 +379,25 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
lang.image = basename
lang.save()
- for language in row['PREFERED WRITEN LANGUAGE'][:1]:
- pref_lang, created = Language.objects.get_or_create(name=language
- ,locale=locale_table.get(language,(None, None))[0])
- if created:
- logging.warn(
- 'New Language (from Prefered) added: {}'.format(language))
- pref_lang.save()
+ languages.append(lang)
+
+ return languages
+
+def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_worker):
+ # Languages
+ if len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
+ logging.warn('No Languages available')
+ elif len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) > 0:
+ row['LANGUAGES'] = row['PREFERED WRITEN LANGUAGE']
+ elif len(row['LANGUAGES']) > 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
+ row['PREFERED WRITEN LANGUAGE'] = row['LANGUAGES']
+
+ languages = create_languages(row['LANGUAGES'])
+ pref_lang = create_languages(row['PREFERED WRITEN LANGUAGE'][:1])
+ if len(pref_lang) > 0:
+ pref_lang = pref_lang[0]
+ elif len(languages) > 0:
+ pref_lang = languages[0]
# Country
country = row['COUNTRY']
@@ -399,15 +435,20 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
# Health Partner
# create health partner (Referral)
- health_partner, created = Worker.objects.get_or_create(name=row['REFERRAL'])
- health_partner.roles.update(role=ROLE_CHOICES_HEALTH_PARTNER)
- # create workerStudyRole
- workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(
- worker=health_partner, study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_HEALTH_PARTNER)
- health_partner.save()
- if created:
- logging.warn('New Health Partner added: {}'.format(row['REFERRAL']))
+ health_partner = None
+ if row['REFERRAL'].strip() != '':
+ logging.warn('Trying to get or create Worker: {}'.format(row['REFERRAL']))
+ health_partner, created = Worker.objects.get_or_create(name=row['REFERRAL'])
+ health_partner.roles.update(role=ROLE_CHOICES_HEALTH_PARTNER)
+ # create workerStudyRole
+ workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(
+ worker=health_partner, study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_HEALTH_PARTNER)
+ health_partner.save()
+ if created:
+ logging.warn('New Health Partner added: {}'.format(row['REFERRAL']))
+ if row['SS NUMBER'] is None:
+ row['SS NUMBER'] = ''
subject, created = Subject.objects.get_or_create(social_security_number=row['SS NUMBER'],
first_name=row['FIRST NAME'],
last_name=row['LAST NAME'],
@@ -466,7 +507,7 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
#VOUCHERS
voucher_references = row['VOUCHER REFERENCE']
for voucher_reference in voucher_references:
- voucher = add_subject_vouchers(voucher_reference, health_partner, voucher_types)
+ voucher = add_subject_vouchers(voucher_reference, health_partner, voucher_types, nd_number)
# Visits
# Consider all visits as part of the same visit with multiple appointments
@@ -478,8 +519,14 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
sum(Ans, []) flattens the resulting list from the map since each findall returns a list
map to convert string to datetime
'''
- visit_dates = map(lambda x: datetime.datetime.strptime(
- x, '%d.%m.%Y'), sum(map(date_regex.findall, row[visit_columns].values), []))
+ def parse_visit_date(date_string):
+ try:
+ dtime = datetime.datetime.strptime(date_string, '%d.%m.%Y %H:%M')
+ except ValueError:
+ dtime = datetime.datetime.strptime(date_string, '%d.%m.%Y')
+ return dtime
+
+ visit_dates = map(lambda x: parse_visit_date(x), sum(map(date_regex.findall, row[visit_columns].values), []))
# get first and last elements of the sorted element
datetime_begin, datetime_end = itemgetter(*[0, -1])(sorted(visit_dates))
@@ -502,8 +549,9 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
'''
starting_hour = 9
for visit_date in set(visit_dates):
- datetime_when = visit_date.replace(hour=starting_hour, minute=0, second=0, microsecond=0)
- starting_hour+=1
+ if visit_date.hour == 0:
+ datetime_when = visit_date.replace(hour=starting_hour, minute=0, second=0, microsecond=0)
+ starting_hour+=1
# get the indices of each occurrence of the date and use them to get
# the appointment types
@@ -528,34 +576,37 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
app_type_link.save()
def createWorker(password, email='', username='admin', first_name='LCSB', last_name='Admin',
- specialization='Technician', unit='LCSB'):
-
- # create user
- defaults = {'email': email, 'password': password}
- user, _ = User.objects.update_or_create(username=username, defaults=defaults)
- user.set_password(password)
- user.is_superuser = True
- user.is_staff = True
- user.is_admin = True
- user.save()
+ specialization='Technician', unit='LCSB', languages=[], role=ROLE_CHOICES_TECHNICIAN,
+ name='', comment=None):
+ user = None
+ if username is not None:
+ # create user
+ defaults = {'email': email, 'password': password}
+ user, _ = User.objects.update_or_create(username=username, defaults=defaults)
+ user.set_password(password)
+ user.is_superuser = True
+ user.is_staff = True
+ user.is_admin = True
+ user.save()
# create worker
defaults = {'first_name': first_name, 'last_name': last_name,
'email': email, 'unit': unit, 'specialization': specialization
- , 'user': user}
+ , 'user': user, 'name': name, 'comment': comment}
+
worker, _ = Worker.objects.update_or_create(first_name=first_name,
last_name=last_name, defaults=defaults)
-
locations = Location.objects.all()
worker.locations.set(locations)
- languages = Language.objects.all()
+ if len(languages) == 0:
+ languages = Language.objects.all()
worker.languages.set(languages)
worker.save()
# create workerStudyRole
workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(worker=worker,
- study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_TECHNICIAN)
+ study_id=GLOBAL_STUDY_ID, role=role)
workerStudyRole.save()
logging.info('SuperUser and Worker {} created'.format(username))
return worker
@@ -570,16 +621,66 @@ if __name__ == '__main__':
logging.warn('Please, execute the program with a valid file path.')
sys.exit(1)
+ admin_password = None
+ if len(sys.argv) >= 3:
+ admin_password = sys.argv[2]
+
#create worker and super user
- pass1 = ''
- pass2 = None
- while pass1 != pass2:
- pass1 = getpass.getpass('Please type a password for the Admin user: ')
- pass2 = getpass.getpass('Please type your password again: ')
- if pass1 != pass2:
- print 'Password mismatch, please try again'
+ if admin_password is None:
+ pass1 = ''
+ pass2 = None
+ while pass1 != pass2:
+ pass1 = getpass.getpass('Please type a password for the Admin user: ')
+ pass2 = getpass.getpass('Please type your password again: ')
+ if pass1 != pass2:
+ print 'Password mismatch, please try again'
+ else:
+ pass1 = admin_password
lcsb_worker = createWorker(pass1)
+ #STUDY
+ study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+ #enable vouchers
+ study.columns.voucher_types = True
+ study.columns.vouchers = True
+ study.nd_number_study_subject_regex = r'^PDP\d{4}$'
+ study.columns.save()
+ study.save()
+
+ #READ WORKERS SHEET
+ df = pd.read_excel(file, dtype=object, sheet_name='Workers')
+ #convert column name to upper case
+ df.columns = [c.upper() for c in df.columns]
+ #convert languages
+ df['LANGUAGES'] = df['LANGUAGES'].apply(apply_column_languages)
+ df['ROLE'] = df['ROLE'].apply(parse_role)
+ for index, row in df.iterrows():
+ languages = create_languages(row['LANGUAGES'])
+ createWorker(row['PASSWORD'], email=row['MAIL'], username=row['USERNAME'], first_name=row['FIRST NAME'], last_name=row['LAST NAME'],
+ specialization=row['ROLE'], role=row['ROLE'], unit='CHL', languages=languages)
+
+ #READ VOUCHER PARTNER SHEET
+ df = pd.read_excel(file, dtype=object, sheet_name='Voucher Partners')
+ #convert column name to upper case
+ df.columns = [c.upper() for c in df.columns]
+ df['ROLE'] = df['ROLE'].apply(parse_role)
+ for index, row in df.iterrows():
+ worker = createWorker(None, email='', username=None, first_name='', last_name='', specialization=row['ROLE'],
+ role=row['ROLE'], unit='CHL', languages=[], name=row['NAME'], comment=row.get('COMMENT', ''))
+ if type(row['OFFER']) != float:
+ offers = row['OFFER'].split(';')
+ voucher_types = []
+ for offer in offers:
+ code = create_voucher_code(offer)
+ voucher_type_codes.add(code)
+ voucher_type, created = VoucherType.objects.update_or_create(description=offer.title(), study=study, code=code)
+ if created:
+ logging.warn(u'Voucher type created: {} ({})'.format(offer, code))
+ voucher_types.append(voucher_type)
+ worker.voucher_types.set(voucher_types)
+ worker.save()
+
+ #READ FIRST SHEET
df = pd.read_excel(file, dtype=object)
df = df.fillna('').astype(unicode)
df.columns = [c.upper() for c in df.columns]
@@ -590,14 +691,6 @@ if __name__ == '__main__':
# get visits columns
regex = re.compile(r'\(V\d\)')
#
- study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
- #enable vouchers
- study.columns.voucher_types = True
- study.columns.vouchers = True
- study.nd_number_study_subject_regex = r'^PDP\d{4}$'
- study.columns.save()
- study.save()
- #
visit_columns = filter(regex.search, df.columns)
assessments = OrderedDict([('Cognitive Test', 180), ('Risk Factor', 120),