From 846c921736cf74ca8db1fbb0691db7e5eecea4cf Mon Sep 17 00:00:00 2001
From: Piotr Gawron <piotr.gawron@uni.lu>
Date: Wed, 18 Nov 2020 15:22:01 +0100
Subject: [PATCH] DATE_FORMAT is part of import_data
---
smash/web/importer/csv_subject_import_reader.py | 6 ++----
.../web/importer/csv_tns_subject_import_reader.py | 6 ++----
smash/web/importer/csv_tns_visit_import_reader.py | 15 ++++++---------
smash/web/migrations/0179_visitimportdata.py | 1 +
smash/web/models/etl/etl.py | 6 ++++++
.../importer/test_csv_subject_import_reader.py | 2 +-
.../test_tns_csv_subject_import_reader.py | 6 ++++--
.../importer/test_tns_csv_visit_import_reader.py | 4 +++-
8 files changed, 25 insertions(+), 21 deletions(-)
diff --git a/smash/web/importer/csv_subject_import_reader.py b/smash/web/importer/csv_subject_import_reader.py
index 74e02edc..9cf314f2 100644
--- a/smash/web/importer/csv_subject_import_reader.py
+++ b/smash/web/importer/csv_subject_import_reader.py
@@ -6,8 +6,6 @@ from typing import List
from web.models import StudySubject, Subject, SubjectImportData
from .subject_import_reader import SubjectImportReader
-CSV_DATE_FORMAT = "%d-%m-%Y"
-
logger = logging.getLogger(__name__)
@@ -58,7 +56,7 @@ class CsvSubjectImportReader(SubjectImportReader):
def get_new_date_value(self, old_value: datetime, column_name: str, new_value: str) -> datetime:
if old_value is None or old_value == "":
try:
- result = datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ result = datetime.datetime.strptime(new_value, self.import_data.date_format)
except ValueError:
logger.warning("Invalid date: " + new_value)
result = old_value
@@ -68,4 +66,4 @@ class CsvSubjectImportReader(SubjectImportReader):
logger.warning(
"Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
"). Latest value will be used")
- return datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ return datetime.datetime.strptime(new_value, self.import_data.date_format)
diff --git a/smash/web/importer/csv_tns_subject_import_reader.py b/smash/web/importer/csv_tns_subject_import_reader.py
index 33f29abe..145307bb 100644
--- a/smash/web/importer/csv_tns_subject_import_reader.py
+++ b/smash/web/importer/csv_tns_subject_import_reader.py
@@ -7,8 +7,6 @@ from typing import List
from web.models import StudySubject, Subject, SubjectImportData
from .subject_import_reader import SubjectImportReader
-CSV_DATE_FORMAT = "%d/%m/%Y"
-
logger = logging.getLogger(__name__)
@@ -80,7 +78,7 @@ class TnsCsvSubjectImportReader(SubjectImportReader):
def get_new_date_value(self, old_value: datetime, column_name: str, new_value: str) -> datetime:
if old_value is None or old_value == "":
try:
- result = datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ result = datetime.datetime.strptime(new_value, self.import_data.date_format)
except ValueError:
logger.warning("Invalid date: " + new_value)
result = old_value
@@ -90,7 +88,7 @@ class TnsCsvSubjectImportReader(SubjectImportReader):
logger.warning(
"Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
"). Latest value will be used")
- return datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ return datetime.datetime.strptime(new_value, self.import_data.date_format)
def remove_bom(line):
diff --git a/smash/web/importer/csv_tns_visit_import_reader.py b/smash/web/importer/csv_tns_visit_import_reader.py
index d3f2d163..039fc39d 100644
--- a/smash/web/importer/csv_tns_visit_import_reader.py
+++ b/smash/web/importer/csv_tns_visit_import_reader.py
@@ -128,7 +128,7 @@ class TnsCsvVisitImportReader:
def get_visit_date(self, data: dict) -> datetime:
try:
- return TnsCsvVisitImportReader.extract_date(data[self.visit_import_data.visit_date_column_name])
+ return self.extract_date(data[self.visit_import_data.visit_date_column_name])
except KeyError as e:
raise EtlException('Visit date is not defined') from e
@@ -152,17 +152,14 @@ class TnsCsvVisitImportReader:
except KeyError as e:
raise EtlException('Subject id is not defined') from e
- @staticmethod
- def extract_date(text: str) -> datetime:
- # by default use day after tomorrow
- result = datetime.datetime.now().replace(hour=9, minute=0) + datetime.timedelta(days=2)
+ def extract_date(self, text: str) -> datetime:
try:
- year = int(text[:4])
- month = int(text[4:6])
- day = int(text[6:8])
- result = result.replace(year=year, month=month, day=day, tzinfo=pytz.UTC)
+ result = datetime.datetime.strptime(text, self.visit_import_data.date_format)
except ValueError:
+ # by default use day after tomorrow
+ result = datetime.datetime.now() + datetime.timedelta(days=2)
logger.warning("Invalid date: " + text)
+ result = result.replace(hour=9, minute=0, tzinfo=pytz.UTC)
return result
def extract_location(self, data: dict) -> Location:
diff --git a/smash/web/migrations/0179_visitimportdata.py b/smash/web/migrations/0179_visitimportdata.py
index 8150b59e..40054594 100644
--- a/smash/web/migrations/0179_visitimportdata.py
+++ b/smash/web/migrations/0179_visitimportdata.py
@@ -18,6 +18,7 @@ class Migration(migrations.Migration):
('filename', models.CharField(blank=True, default='', max_length=128, verbose_name='File used for automatic import')),
('run_at_times', models.CharField(blank=True, default='', max_length=1024, verbose_name='At what time automatic import should run')),
('csv_delimiter', models.CharField(blank=False, default=',', max_length=1, verbose_name='CSV delimiter')),
+ ('date_format', models.CharField(blank=False, default='%Y-%m-%d', max_length=20, verbose_name='Date format')),
],
),
migrations.CreateModel(
diff --git a/smash/web/models/etl/etl.py b/smash/web/models/etl/etl.py
index 6d548acd..26ba15af 100644
--- a/smash/web/models/etl/etl.py
+++ b/smash/web/models/etl/etl.py
@@ -42,6 +42,12 @@ class EtlData(models.Model):
null=False,
blank=False
)
+ date_format = models.CharField(max_length=20,
+ verbose_name='Date format',
+ default='%Y-%m-%d',
+ null=False,
+ blank=False
+ )
def file_available(self) -> bool:
if self.filename is None or self.filename == '':
diff --git a/smash/web/tests/importer/test_csv_subject_import_reader.py b/smash/web/tests/importer/test_csv_subject_import_reader.py
index 4596154f..29cf8bce 100644
--- a/smash/web/tests/importer/test_csv_subject_import_reader.py
+++ b/smash/web/tests/importer/test_csv_subject_import_reader.py
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
class TestCsvReader(TestCase):
def setUp(self):
- self.study_import_data = SubjectImportData.objects.create(study=get_test_study())
+ self.study_import_data = SubjectImportData.objects.create(study=get_test_study(), date_format="%d-%m-%Y")
def test_load_data(self):
self.study_import_data.filename = get_resource_path('import.csv')
diff --git a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
index 28e5bad5..d47d4adc 100644
--- a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
+++ b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
@@ -15,7 +15,9 @@ logger = logging.getLogger(__name__)
class TestTnsCsvSubjectReader(TestCase):
def setUp(self):
- self.study_import_data = SubjectImportData.objects.create(study=get_test_study())
+ self.study_import_data = SubjectImportData.objects.create(study=get_test_study(),
+ date_format="%d/%m/%Y",
+ csv_delimiter=";")
self.warning_counter = MsgCounterHandler()
logging.getLogger('').addHandler(self.warning_counter)
@@ -24,7 +26,7 @@ class TestTnsCsvSubjectReader(TestCase):
def test_load_data(self):
self.study_import_data.filename = get_resource_path('tns_subjects_import.csv')
- study_subjects = TnsCsvSubjectImportReader(self.study_import_data).load_data(self.study_import_data.filename)
+ study_subjects = TnsCsvSubjectImportReader(self.study_import_data).load_data()
self.assertEqual(3, len(study_subjects))
study_subject = study_subjects[1]
self.assertEqual("John2", study_subject.subject.first_name)
diff --git a/smash/web/tests/importer/test_tns_csv_visit_import_reader.py b/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
index c65ff64c..74211d09 100644
--- a/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
+++ b/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
@@ -26,7 +26,9 @@ class TestTnsCsvVisitReader(TestCase):
settings.ETL_ROOT = os.path.dirname(get_resource_path('tns_vouchers_import.csv'))
self.visit_import_data = VisitImportData.objects.create(study=get_test_study(),
appointment_type=appointment_type,
- import_worker=create_worker())
+ import_worker=create_worker(),
+ date_format="%Y%m%d",
+ csv_delimiter=";")
self.warning_counter = MsgCounterHandler()
logging.getLogger('').addHandler(self.warning_counter)
--
GitLab