Skip to content
Snippets Groups Projects
Commit a4312ac7 authored by Carlos Vega's avatar Carlos Vega
Browse files

updated script to support new sheets in excel and extra information

parent 39da4506
No related branches found
No related tags found
3 merge requests!207Merge Devel 0.12.x into Master (0.13.x),!203Pdp/import data,!200Improvement/pdp import data and button to export subjects table to excel
Pipeline #7736 failed
......@@ -25,7 +25,7 @@ from web.utils import is_valid_social_security_number
from web.models import VoucherType, Voucher, Country, AppointmentTypeLink, AppointmentType, Study, Worker, Language, Subject, WorkerStudyRole, StudySubject, Location, FlyingTeam, Visit, Appointment, AppointmentType
from web.models.worker_study_role import ROLE_CHOICES_TECHNICIAN, WORKER_STAFF, ROLE_CHOICES_SECRETARY, ROLE_CHOICES_HEALTH_PARTNER, \
WORKER_HEALTH_PARTNER, ROLE_CHOICES_VOUCHER_PARTNER, WORKER_VOUCHER_PARTNER
WORKER_HEALTH_PARTNER, ROLE_CHOICES_DOCTOR, ROLE_CHOICES_VOUCHER_PARTNER, ROLE_CHOICES, ROLE_CHOICES_NURSE, ROLE_CHOICES_PSYCHOLOGIST, ROLE_CHOICES_PROJECT_MANAGER
DEFAULT_LOCATION = 'CHL'
DEFAULT_LOCATION_PREFIX = 'P'
......@@ -149,7 +149,13 @@ def parse_boolean(boolean_Y_N):
def parse_column_date_of_birth(date):
return datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y-%m-%d')
try:
return datetime.datetime.strptime(date, '%d.%m.%Y %H:%M').strftime('%Y-%m-%d')
except ValueError:
try:
return datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y-%m-%d')
except:
return None
# gender
gender_table = {'m': SEX_CHOICES_MALE, 'f': SEX_CHOICES_FEMALE}
......@@ -179,6 +185,18 @@ def parse_column_ss_number(ss):
logging.debug('Invalid SS number: (Length not valid) |{}|'.format(ss))
return ss
def parse_role(role_name):
d = {
'NURSE': ROLE_CHOICES_NURSE,
'PSYCHOLOGIST': ROLE_CHOICES_PSYCHOLOGIST,
'SECRETARY': ROLE_CHOICES_SECRETARY,
'PROJECT MANAGER': ROLE_CHOICES_PROJECT_MANAGER,
'DOCTOR': ROLE_CHOICES_DOCTOR,
'HEALTH_PARTNER': ROLE_CHOICES_HEALTH_PARTNER,
'VOUCHER_PARTNER': ROLE_CHOICES_VOUCHER_PARTNER
}
return d[role_name] #crash if no valid role_name is given
# Language
......@@ -188,6 +206,7 @@ language_table = {
'IT': 'Italian',
'F': 'French',
'D': 'German',
'G': 'German',
'E': 'English',
'P': 'Portuguese',
'A': 'Arabic',
......@@ -278,7 +297,7 @@ converters = {
# add voucher for subject
voucher_partners = {}
voucher_partners['ZIT'] = 'Zitha'
def add_subject_vouchers(voucher_reference, referral, voucher_types):
def add_subject_vouchers(voucher_reference, referral, voucher_types, subject_nd_number):
nd_number, date, voucher_partner, voucher_type, num = voucher_reference.split('-')
nd_number = nd_number.upper().replace('ND', 'PDP')
issue_date = datetime.datetime.strptime(date, '%Y%m%d')
......@@ -297,7 +316,9 @@ def add_subject_vouchers(voucher_reference, referral, voucher_types):
vt = VoucherType.objects.get(code=voucher_type)
study_subject = StudySubject.objects.get(nd_number=nd_number)
if nd_number != subject_nd_number:
logging.warn('voucher reference nd_number is not the same! {} != {}'.format(nd_number, subject_nd_number))
study_subject = StudySubject.objects.get(nd_number=subject_nd_number)
voucher, created = Voucher.objects.update_or_create(number=voucher_reference, issue_date=issue_date,
expiry_date=expiry_date, voucher_type=vt, study_subject=study_subject,
......@@ -305,8 +326,21 @@ def add_subject_vouchers(voucher_reference, referral, voucher_types):
logging.warn('New Voucher added: {}'.format(voucher_reference))
return voucher
# create voucher types
voucher_type_codes = set()
def create_voucher_code(description):
code = re.sub(r'\([^)]*\)', '', description.strip())
code = re.sub(r'[aeiou]+', '', code.lower()).strip()
code = ''.join([e.capitalize() for e in code.split(' ')[:2]])
while code in voucher_type_codes:
if re.match(r'.*_\d+$', code) is None:
code = code + '_2'
else:
spt = code.split('_')
num = int(spt[-1])+1
code = ''.join(spt[:len(spt)]) + str(num)
return code
def create_voucher_types(voucher_types_dict, study):
voucher_types = {}
for name, code in voucher_types_dict.items():
......@@ -325,21 +359,11 @@ def create_appointment_types(assessments):
appointmentTypes.append(appointmentType)
return appointmentTypes
def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_worker):
# Languages
if len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
logging.warn('No Languages available')
elif len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) > 0:
row['LANGUAGES'] = row['PREFERED WRITEN LANGUAGE']
elif len(row['LANGUAGES']) > 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
row['PREFERED WRITEN LANGUAGE'] = row['LANGUAGES']
def create_languages(languages_cell):
languages = []
for language in row['LANGUAGES']:
for language in languages_cell:
lang, created = Language.objects.get_or_create(
name=language, locale=locale_table.get(language,(None, None))[0])
languages.append(lang)
if created:
logging.warn('New Language added: {}'.format(language))
......@@ -355,13 +379,25 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
lang.image = basename
lang.save()
for language in row['PREFERED WRITEN LANGUAGE'][:1]:
pref_lang, created = Language.objects.get_or_create(name=language
,locale=locale_table.get(language,(None, None))[0])
if created:
logging.warn(
'New Language (from Prefered) added: {}'.format(language))
pref_lang.save()
languages.append(lang)
return languages
def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_worker):
# Languages
if len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
logging.warn('No Languages available')
elif len(row['LANGUAGES']) == 0 and len(row['PREFERED WRITEN LANGUAGE']) > 0:
row['LANGUAGES'] = row['PREFERED WRITEN LANGUAGE']
elif len(row['LANGUAGES']) > 0 and len(row['PREFERED WRITEN LANGUAGE']) == 0:
row['PREFERED WRITEN LANGUAGE'] = row['LANGUAGES']
languages = create_languages(row['LANGUAGES'])
pref_lang = create_languages(row['PREFERED WRITEN LANGUAGE'][:1])
if len(pref_lang) > 0:
pref_lang = pref_lang[0]
elif len(languages) > 0:
pref_lang = languages[0]
# Country
country = row['COUNTRY']
......@@ -399,15 +435,20 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
# Health Partner
# create health partner (Referral)
health_partner, created = Worker.objects.get_or_create(name=row['REFERRAL'])
health_partner.roles.update(role=ROLE_CHOICES_HEALTH_PARTNER)
# create workerStudyRole
workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(
worker=health_partner, study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_HEALTH_PARTNER)
health_partner.save()
if created:
logging.warn('New Health Partner added: {}'.format(row['REFERRAL']))
health_partner = None
if row['REFERRAL'].strip() != '':
logging.warn('Trying to get or create Worker: {}'.format(row['REFERRAL']))
health_partner, created = Worker.objects.get_or_create(name=row['REFERRAL'])
health_partner.roles.update(role=ROLE_CHOICES_HEALTH_PARTNER)
# create workerStudyRole
workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(
worker=health_partner, study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_HEALTH_PARTNER)
health_partner.save()
if created:
logging.warn('New Health Partner added: {}'.format(row['REFERRAL']))
if row['SS NUMBER'] is None:
row['SS NUMBER'] = ''
subject, created = Subject.objects.get_or_create(social_security_number=row['SS NUMBER'],
first_name=row['FIRST NAME'],
last_name=row['LAST NAME'],
......@@ -466,7 +507,7 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
#VOUCHERS
voucher_references = row['VOUCHER REFERENCE']
for voucher_reference in voucher_references:
voucher = add_subject_vouchers(voucher_reference, health_partner, voucher_types)
voucher = add_subject_vouchers(voucher_reference, health_partner, voucher_types, nd_number)
# Visits
# Consider all visits as part of the same visit with multiple appointments
......@@ -478,8 +519,14 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
sum(Ans, []) flattens the resulting list from the map since each findall returns a list
map to convert string to datetime
'''
visit_dates = map(lambda x: datetime.datetime.strptime(
x, '%d.%m.%Y'), sum(map(date_regex.findall, row[visit_columns].values), []))
def parse_visit_date(date_string):
try:
dtime = datetime.datetime.strptime(date_string, '%d.%m.%Y %H:%M')
except ValueError:
dtime = datetime.datetime.strptime(date_string, '%d.%m.%Y')
return dtime
visit_dates = map(lambda x: parse_visit_date(x), sum(map(date_regex.findall, row[visit_columns].values), []))
# get first and last elements of the sorted element
datetime_begin, datetime_end = itemgetter(*[0, -1])(sorted(visit_dates))
......@@ -502,8 +549,9 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
'''
starting_hour = 9
for visit_date in set(visit_dates):
datetime_when = visit_date.replace(hour=starting_hour, minute=0, second=0, microsecond=0)
starting_hour+=1
if visit_date.hour == 0:
datetime_when = visit_date.replace(hour=starting_hour, minute=0, second=0, microsecond=0)
starting_hour+=1
# get the indices of each occurrence of the date and use them to get
# the appointment types
......@@ -528,34 +576,37 @@ def parse_row(index, row, visit_columns, appointmentTypes, voucher_types, lcsb_w
app_type_link.save()
def createWorker(password, email='', username='admin', first_name='LCSB', last_name='Admin',
specialization='Technician', unit='LCSB'):
# create user
defaults = {'email': email, 'password': password}
user, _ = User.objects.update_or_create(username=username, defaults=defaults)
user.set_password(password)
user.is_superuser = True
user.is_staff = True
user.is_admin = True
user.save()
specialization='Technician', unit='LCSB', languages=[], role=ROLE_CHOICES_TECHNICIAN,
name='', comment=None):
user = None
if username is not None:
# create user
defaults = {'email': email, 'password': password}
user, _ = User.objects.update_or_create(username=username, defaults=defaults)
user.set_password(password)
user.is_superuser = True
user.is_staff = True
user.is_admin = True
user.save()
# create worker
defaults = {'first_name': first_name, 'last_name': last_name,
'email': email, 'unit': unit, 'specialization': specialization
, 'user': user}
, 'user': user, 'name': name, 'comment': comment}
worker, _ = Worker.objects.update_or_create(first_name=first_name,
last_name=last_name, defaults=defaults)
locations = Location.objects.all()
worker.locations.set(locations)
languages = Language.objects.all()
if len(languages) == 0:
languages = Language.objects.all()
worker.languages.set(languages)
worker.save()
# create workerStudyRole
workerStudyRole, _ = WorkerStudyRole.objects.update_or_create(worker=worker,
study_id=GLOBAL_STUDY_ID, role=ROLE_CHOICES_TECHNICIAN)
study_id=GLOBAL_STUDY_ID, role=role)
workerStudyRole.save()
logging.info('SuperUser and Worker {} created'.format(username))
return worker
......@@ -570,16 +621,66 @@ if __name__ == '__main__':
logging.warn('Please, execute the program with a valid file path.')
sys.exit(1)
admin_password = None
if len(sys.argv) >= 3:
admin_password = sys.argv[2]
#create worker and super user
pass1 = ''
pass2 = None
while pass1 != pass2:
pass1 = getpass.getpass('Please type a password for the Admin user: ')
pass2 = getpass.getpass('Please type your password again: ')
if pass1 != pass2:
print 'Password mismatch, please try again'
if admin_password is None:
pass1 = ''
pass2 = None
while pass1 != pass2:
pass1 = getpass.getpass('Please type a password for the Admin user: ')
pass2 = getpass.getpass('Please type your password again: ')
if pass1 != pass2:
print 'Password mismatch, please try again'
else:
pass1 = admin_password
lcsb_worker = createWorker(pass1)
#STUDY
study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
#enable vouchers
study.columns.voucher_types = True
study.columns.vouchers = True
study.nd_number_study_subject_regex = r'^PDP\d{4}$'
study.columns.save()
study.save()
#READ WORKERS SHEET
df = pd.read_excel(file, dtype=object, sheet_name='Workers')
#convert column name to upper case
df.columns = [c.upper() for c in df.columns]
#convert languages
df['LANGUAGES'] = df['LANGUAGES'].apply(apply_column_languages)
df['ROLE'] = df['ROLE'].apply(parse_role)
for index, row in df.iterrows():
languages = create_languages(row['LANGUAGES'])
createWorker(row['PASSWORD'], email=row['MAIL'], username=row['USERNAME'], first_name=row['FIRST NAME'], last_name=row['LAST NAME'],
specialization=row['ROLE'], role=row['ROLE'], unit='CHL', languages=languages)
#READ VOUCHER PARTNER SHEET
df = pd.read_excel(file, dtype=object, sheet_name='Voucher Partners')
#convert column name to upper case
df.columns = [c.upper() for c in df.columns]
df['ROLE'] = df['ROLE'].apply(parse_role)
for index, row in df.iterrows():
worker = createWorker(None, email='', username=None, first_name='', last_name='', specialization=row['ROLE'],
role=row['ROLE'], unit='CHL', languages=[], name=row['NAME'], comment=row.get('COMMENT', ''))
if type(row['OFFER']) != float:
offers = row['OFFER'].split(';')
voucher_types = []
for offer in offers:
code = create_voucher_code(offer)
voucher_type_codes.add(code)
voucher_type, created = VoucherType.objects.update_or_create(description=offer.title(), study=study, code=code)
if created:
logging.warn(u'Voucher type created: {} ({})'.format(offer, code))
voucher_types.append(voucher_type)
worker.voucher_types.set(voucher_types)
worker.save()
#READ FIRST SHEET
df = pd.read_excel(file, dtype=object)
df = df.fillna('').astype(unicode)
df.columns = [c.upper() for c in df.columns]
......@@ -590,14 +691,6 @@ if __name__ == '__main__':
# get visits columns
regex = re.compile(r'\(V\d\)')
#
study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
#enable vouchers
study.columns.voucher_types = True
study.columns.vouchers = True
study.nd_number_study_subject_regex = r'^PDP\d{4}$'
study.columns.save()
study.save()
#
visit_columns = filter(regex.search, df.columns)
assessments = OrderedDict([('Cognitive Test', 180), ('Risk Factor', 120),
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment