diff --git a/smash/web/importer/csv_subject_import_reader.py b/smash/web/importer/csv_subject_import_reader.py
index 74e02edc480b91bc52771faa28063051d9409dc7..9cf314f254985018a44c5b71b4012c232810f6c5 100644
--- a/smash/web/importer/csv_subject_import_reader.py
+++ b/smash/web/importer/csv_subject_import_reader.py
@@ -6,8 +6,6 @@ from typing import List
from web.models import StudySubject, Subject, SubjectImportData
from .subject_import_reader import SubjectImportReader
-CSV_DATE_FORMAT = "%d-%m-%Y"
-
logger = logging.getLogger(__name__)
@@ -58,7 +56,7 @@ class CsvSubjectImportReader(SubjectImportReader):
def get_new_date_value(self, old_value: datetime, column_name: str, new_value: str) -> datetime:
if old_value is None or old_value == "":
try:
- result = datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ result = datetime.datetime.strptime(new_value, self.import_data.date_format)
except ValueError:
logger.warning("Invalid date: " + new_value)
result = old_value
@@ -68,4 +66,4 @@ class CsvSubjectImportReader(SubjectImportReader):
logger.warning(
"Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
"). Latest value will be used")
- return datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ return datetime.datetime.strptime(new_value, self.import_data.date_format)
diff --git a/smash/web/importer/csv_tns_subject_import_reader.py b/smash/web/importer/csv_tns_subject_import_reader.py
index 33f29abef9654c78908f8ae128bf29f2efeb5ae3..145307bb3497ab838fea13cc1e612539d954834e 100644
--- a/smash/web/importer/csv_tns_subject_import_reader.py
+++ b/smash/web/importer/csv_tns_subject_import_reader.py
@@ -7,8 +7,6 @@ from typing import List
from web.models import StudySubject, Subject, SubjectImportData
from .subject_import_reader import SubjectImportReader
-CSV_DATE_FORMAT = "%d/%m/%Y"
-
logger = logging.getLogger(__name__)
@@ -80,7 +78,7 @@ class TnsCsvSubjectImportReader(SubjectImportReader):
def get_new_date_value(self, old_value: datetime, column_name: str, new_value: str) -> datetime:
if old_value is None or old_value == "":
try:
- result = datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ result = datetime.datetime.strptime(new_value, self.import_data.date_format)
except ValueError:
logger.warning("Invalid date: " + new_value)
result = old_value
@@ -90,7 +88,7 @@ class TnsCsvSubjectImportReader(SubjectImportReader):
logger.warning(
"Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
"). Latest value will be used")
- return datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ return datetime.datetime.strptime(new_value, self.import_data.date_format)
def remove_bom(line):
diff --git a/smash/web/importer/csv_tns_visit_import_reader.py b/smash/web/importer/csv_tns_visit_import_reader.py
index d3f2d163d7ea2ab332e7307fad42126a1950b1f6..039fc39dfb0f92494aae20c1fcf9c5091e2fed49 100644
--- a/smash/web/importer/csv_tns_visit_import_reader.py
+++ b/smash/web/importer/csv_tns_visit_import_reader.py
@@ -128,7 +128,7 @@ class TnsCsvVisitImportReader:
def get_visit_date(self, data: dict) -> datetime:
try:
- return TnsCsvVisitImportReader.extract_date(data[self.visit_import_data.visit_date_column_name])
+ return self.extract_date(data[self.visit_import_data.visit_date_column_name])
except KeyError as e:
raise EtlException('Visit date is not defined') from e
@@ -152,17 +152,14 @@ class TnsCsvVisitImportReader:
except KeyError as e:
raise EtlException('Subject id is not defined') from e
- @staticmethod
- def extract_date(text: str) -> datetime:
- # by default use day after tomorrow
- result = datetime.datetime.now().replace(hour=9, minute=0) + datetime.timedelta(days=2)
+ def extract_date(self, text: str) -> datetime:
try:
- year = int(text[:4])
- month = int(text[4:6])
- day = int(text[6:8])
- result = result.replace(year=year, month=month, day=day, tzinfo=pytz.UTC)
+ result = datetime.datetime.strptime(text, self.visit_import_data.date_format)
except ValueError:
+ # by default use day after tomorrow
+ result = datetime.datetime.now() + datetime.timedelta(days=2)
logger.warning("Invalid date: " + text)
+ result = result.replace(hour=9, minute=0, tzinfo=pytz.UTC)
return result
def extract_location(self, data: dict) -> Location:
diff --git a/smash/web/migrations/0179_visitimportdata.py b/smash/web/migrations/0179_visitimportdata.py
index 8150b59ed6b16c8922d8107cd1a4ac72bb66b1f1..4005459479c4a677546997200cea023a7dd1a1e4 100644
--- a/smash/web/migrations/0179_visitimportdata.py
+++ b/smash/web/migrations/0179_visitimportdata.py
@@ -18,6 +18,7 @@ class Migration(migrations.Migration):
('filename', models.CharField(blank=True, default='', max_length=128, verbose_name='File used for automatic import')),
('run_at_times', models.CharField(blank=True, default='', max_length=1024, verbose_name='At what time automatic import should run')),
('csv_delimiter', models.CharField(blank=False, default=',', max_length=1, verbose_name='CSV delimiter')),
+ ('date_format', models.CharField(blank=False, default='%Y-%m-%d', max_length=20, verbose_name='Date format')),
],
),
migrations.CreateModel(
diff --git a/smash/web/models/etl/etl.py b/smash/web/models/etl/etl.py
index 6d548acdb205ab754c4cdd11473db249fdb20297..26ba15af0f6e05232ed32c064814ca30914de3aa 100644
--- a/smash/web/models/etl/etl.py
+++ b/smash/web/models/etl/etl.py
@@ -42,6 +42,12 @@ class EtlData(models.Model):
null=False,
blank=False
)
+ date_format = models.CharField(max_length=20,
+ verbose_name='Date format',
+ default='%Y-%m-%d',
+ null=False,
+ blank=False
+ )
def file_available(self) -> bool:
if self.filename is None or self.filename == '':
diff --git a/smash/web/tests/importer/test_csv_subject_import_reader.py b/smash/web/tests/importer/test_csv_subject_import_reader.py
index 4596154f3987134d845be7c56bfac6cdebedf37e..29cf8bceca9fb4ec9c4dc33f9dab42357e69a0b0 100644
--- a/smash/web/tests/importer/test_csv_subject_import_reader.py
+++ b/smash/web/tests/importer/test_csv_subject_import_reader.py
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
class TestCsvReader(TestCase):
def setUp(self):
- self.study_import_data = SubjectImportData.objects.create(study=get_test_study())
+ self.study_import_data = SubjectImportData.objects.create(study=get_test_study(), date_format="%d-%m-%Y")
def test_load_data(self):
self.study_import_data.filename = get_resource_path('import.csv')
diff --git a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
index 28e5bad53992161a12f391cbc1ee8c7242d59a65..d47d4adcd7a9c52aee3e9d876082699a0d1a12e1 100644
--- a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
+++ b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
@@ -15,7 +15,9 @@ logger = logging.getLogger(__name__)
class TestTnsCsvSubjectReader(TestCase):
def setUp(self):
- self.study_import_data = SubjectImportData.objects.create(study=get_test_study())
+ self.study_import_data = SubjectImportData.objects.create(study=get_test_study(),
+ date_format="%d/%m/%Y",
+ csv_delimiter=";")
self.warning_counter = MsgCounterHandler()
logging.getLogger('').addHandler(self.warning_counter)
@@ -24,7 +26,7 @@ class TestTnsCsvSubjectReader(TestCase):
def test_load_data(self):
self.study_import_data.filename = get_resource_path('tns_subjects_import.csv')
- study_subjects = TnsCsvSubjectImportReader(self.study_import_data).load_data(self.study_import_data.filename)
+ study_subjects = TnsCsvSubjectImportReader(self.study_import_data).load_data()
self.assertEqual(3, len(study_subjects))
study_subject = study_subjects[1]
self.assertEqual("John2", study_subject.subject.first_name)
diff --git a/smash/web/tests/importer/test_tns_csv_visit_import_reader.py b/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
index c65ff64c08633480ff60f6e372d9812f780295f4..74211d094183aa84b440a608704cf315ad2e2eb0 100644
--- a/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
+++ b/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
@@ -26,7 +26,9 @@ class TestTnsCsvVisitReader(TestCase):
settings.ETL_ROOT = os.path.dirname(get_resource_path('tns_vouchers_import.csv'))
self.visit_import_data = VisitImportData.objects.create(study=get_test_study(),
appointment_type=appointment_type,
- import_worker=create_worker())
+ import_worker=create_worker(),
+ date_format="%Y%m%d",
+ csv_delimiter=";")
self.warning_counter = MsgCounterHandler()
logging.getLogger('').addHandler(self.warning_counter)