Merge branch 'new-format-for-export' into 'master'

feat: use new format for export, added datetime to log entries and hyphen in use restrictions

See merge request pinar.alper/metadata-tools!7

.gitignore

@@ -3,3 +3,6 @@ __pycache__/
 # Visual studio code
 .vscode
 project_venv
+env
+*.egg-info
+*.log

metadata_tools/importxls/dish_xls_exporter.py

@@ -8,7 +8,10 @@ from .export_utils import get_partners_from_daisy, process_possible_date, proces
 class DishXlsExporter:
 
     def __init__(self):
-        logging.basicConfig(filename='export_dishxls.log', level=logging.DEBUG)
+        logging.basicConfig(filename='export_dishxls.log', 
+                            level=logging.DEBUG,
+                            format='%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s',
+                            datefmt='%Y-%m-%d %H:%M:%S')
         institutions = get_partners_from_daisy()
         self.inst_dict = {}
         self.inst_ac_dict = {}
@@ -106,7 +109,7 @@ class DishXlsExporter:
                     datadec_dict["consent_status_description"] = sheet[33, 1]
 
                     if sheet[20, 1]:
-                        datadec_dict['subject_categories'] = sheet[20, 1].replace(' & ', '_and_')
+                        datadec_dict['subjects_category'] = sheet[20, 1].replace(' & ', '_and_')
 
                     if sheet[12, 1]:
                         lb_code = self.extract_lb_code(sheet[12, 1])
@@ -145,19 +148,19 @@ class DishXlsExporter:
 
                     use_restrictions = []
                     if process_yes_no_dontknow_answer(sheet[24, 1]):
-                        use_restrictions.append({'use_class': 'RS[XX]',
+                        use_restrictions.append({'use_class': 'RS-[XX]',
                                                  'use_restriction_rule': "CONSTRAINTS",
                                                  'use_class_note': sheet[25, 1]})
                     elif process_yes_no_dontknow_answer(sheet[24, 1]) is not None:
-                        use_restrictions.append({'use_class': 'RS[XX]',
+                        use_restrictions.append({'use_class': 'RS-[XX]',
                                                  'use_restriction_rule': "NO_CONSTRAINTS",
                                                  'use_class_note': sheet[25, 1]})
                     if process_yes_no_dontknow_answer(sheet[26, 1]):
-                        use_restrictions.append({'use_class': 'GS[XX]',
+                        use_restrictions.append({'use_class': 'GS-[XX]',
                                                  'use_restriction_rule': "CONSTRAINTS",
                                                  'use_class_note': sheet[27, 1]})
                     elif process_yes_no_dontknow_answer(sheet[26, 1]) is not None:
-                        use_restrictions.append({'use_class': 'GS[XX]',
+                        use_restrictions.append({'use_class': 'GS-[XX]',
                                                  'use_restriction_rule': "NO_CONSTRAINTS",
                                                  'use_class_note': sheet[27, 1]})
 

metadata_tools/importxls/export_utils.py

-from datetime import datetime
 import datetime as dt
 import json
+import logging
 import urllib
+
+from datetime import datetime
 from json import dumps
-from urllib.error import HTTPError, URLError
 from socket import timeout
-import logging
+from urllib.error import HTTPError, URLError
+
 
 def process_yes_no_answer(answer):
     """
@@ -107,7 +109,6 @@ def get_lines_from_string(a_string):
 
 
 def get_partners_from_daisy():
-
     entities_json_str = None
     try:
         urlEntities = urllib.parse.urljoin('https://daisy.lcsb.uni.lu/api/', 'partners')
@@ -130,3 +131,15 @@ def get_partners_from_daisy():
             entities_json_str = dumps('{}')
 
     return json.loads(entities_json_str)
+
+
+def save_exported_datasets_to_file(exported_dataset, output_file):
+    if isinstance(exported_dataset, list):
+        items = exported_dataset
+    else:
+        items = [exported_dataset]
+    obj = {
+        "$schema": "https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/elu-dataset.json",
+        "items": items
+    }
+    return json.dump(obj, output_file, indent=4)

metadata_tools/resources/elu-dataset.json

@@ -145,6 +145,7 @@
                   "US",
                   "PS",
                   "IS",
+                  "IP",
                   "Other"
                 ]
               },

tests/importxls/test_dish_xls_exporter.py

-from unittest import TestCase
-import os
-from metadata_tools.importxls.dish_xls_exporter import DishXlsExporter
 import json
+import os
+
 from pathlib import Path
+from unittest import TestCase
 
-class TestDishXlsPartnerExtractor(TestCase):
+from metadata_tools.importxls.export_utils import save_exported_datasets_to_file
+from metadata_tools.importxls.dish_xls_exporter import DishXlsExporter
 
-    def test_dish_export(self):
 
+class TestDishXlsPartnerExtractor(TestCase):
+    def test_dish_export(self):
         exporter = DishXlsExporter()
         #full_file_path = os.path.join(os.path.dirname(__file__), 'resources', 'UL_Extended_Data_Information_Sheet-8.xlsx')
 
@@ -18,7 +20,7 @@ class TestDishXlsPartnerExtractor(TestCase):
                         full_file_path = os.path.join(dirName, fname)
                         dataset_dict = exporter.export_submission(full_file_path)
                         with open(Path(full_file_path).stem + ".json", 'w') as outfile:
-                            json.dump([dataset_dict], outfile, indent=4)
+                            save_exported_datasets_to_file(dataset_dict, outfile)
                     except ValueError:
                         print("Could not import {}".format(full_file_path))
         return