diff --git a/smash/web/importer/csv_subject_import_reader.py b/smash/web/importer/csv_subject_import_reader.py index 44a30f6f50d9609d69a9e09e9af1ca6ba58c782f..7acb8d5cfd787158ed7540c8a231cf3b68e375c7 100644 --- a/smash/web/importer/csv_subject_import_reader.py +++ b/smash/web/importer/csv_subject_import_reader.py @@ -1,15 +1,13 @@ import csv -import datetime import logging from typing import List, Type, Tuple -import pytz from django.db import models from django.db.models import Field from web.models import StudySubject, Subject, SubjectImportData -from .subject_import_reader import SubjectImportReader from .etl_common import EtlCommon +from .subject_import_reader import SubjectImportReader logger = logging.getLogger(__name__) @@ -53,11 +51,11 @@ class CsvSubjectImportReader(SubjectImportReader): if table == Subject: old_val = getattr(study_subject.subject, field.name) - setattr(study_subject.subject, field.name, self.get_new_value(old_val, field.name, value)) + setattr(study_subject.subject, field.name, self.get_new_value(old_val, value)) elif table == StudySubject: old_val = getattr(study_subject, field.name) print(field.name + ": " + str(old_val) + " - " + str(value)) - setattr(study_subject, field.name, self.get_new_value(old_val, field.name, value)) + setattr(study_subject, field.name, self.get_new_value(old_val, value)) else: logger.warning("Don't know how to handle column " + column_name + " with data " + value) @@ -76,12 +74,3 @@ class CsvSubjectImportReader(SubjectImportReader): found = True if not found: self.mappings[field.name] = object_type, field - - def get_date(self, value): - try: - result = datetime.datetime.strptime(value, self.etl_data.date_format) - result.replace(tzinfo=pytz.UTC) - return result - except ValueError: - logger.warning("Invalid date: " + value) - return None diff --git a/smash/web/importer/etl_common.py b/smash/web/importer/etl_common.py index b18f78aed1cecfc000eedfc0c12dfa88fcba8e84..7538e8c37f176817c938f866246282c708fe5b3d 100644 --- a/smash/web/importer/etl_common.py +++ b/smash/web/importer/etl_common.py @@ -3,6 +3,7 @@ import datetime import logging from typing import Type, Optional +import pytz from django.db import models from web.models import Provenance @@ -15,23 +16,17 @@ class EtlCommon: def __init__(self, import_data: EtlData): self.etl_data = import_data - def get_new_date_value(self, old_value: datetime, column_name: str, new_value: str) -> datetime: - if old_value is None or old_value == "": - try: - result = datetime.datetime.strptime(new_value, self.etl_data.date_format) - except ValueError: - logger.warning("Invalid date: " + new_value) - result = old_value + def get_date(self, value): + try: + result = datetime.datetime.strptime(value, self.etl_data.date_format) + result.replace(tzinfo=pytz.UTC) return result - if new_value is None or new_value == "": - return old_value - logger.warning( - "Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value + - "). Latest value will be used") - return datetime.datetime.strptime(new_value, self.etl_data.date_format) + except ValueError: + logger.warning("Invalid date: " + value) + return None @staticmethod - def get_new_value(old_value: object, column_name: str, new_value: object) -> object: + def get_new_value(old_value: object, new_value: object) -> object: if old_value is None or old_value == "": return new_value if new_value is None or new_value == "": @@ -43,9 +38,6 @@ class EtlCommon: return old_value if old_value in new_value: return new_value - logger.warning( - "Contradicting entries in csv file for column: " + column_name + "(" + str(new_value) + "," + str( - old_value) + "). Latest value will be used") return new_value def create_provenance_and_change_data(self, object_to_change: models.Model, field_name: str, new_value: object, diff --git a/smash/web/importer/importer.py b/smash/web/importer/importer.py index 28b33242fc94d6e35f76968705dc0d997b0acf19..84f68094ad43291e67a2d1db4165b0a2eff480d5 100644 --- a/smash/web/importer/importer.py +++ b/smash/web/importer/importer.py @@ -57,7 +57,7 @@ class Importer(EtlCommon): field.get_internal_type() == "DateField" or \ field.get_internal_type() is "BooleanField": old_value = getattr(db_study_subject.subject, field.name) - new_value = self.get_new_value(old_value, field.name, getattr(study_subject.subject, field.name)) + new_value = self.get_new_value(old_value, getattr(study_subject.subject, field.name)) self.create_provenance_and_change_data(db_study_subject.subject, field.name, new_value, Subject) db_study_subject.subject.save() @@ -66,7 +66,7 @@ class Importer(EtlCommon): field.get_internal_type() == "DateField" or \ field.get_internal_type() is "BooleanField": old_value = getattr(db_study_subject, field.name) - new_value = self.get_new_value(old_value, field.name, getattr(study_subject, field.name)) + new_value = self.get_new_value(old_value, getattr(study_subject, field.name)) self.create_provenance_and_change_data(db_study_subject, field.name, new_value, StudySubject) db_study_subject.save()