core/importer/datasets_exporter.py

@@ -5,12 +5,19 @@ from core.models import  Dataset
 from core.utils import DaisyLogger
 from django.conf import settings
 from io import StringIO
+from urllib.parse import urljoin
+
+JSONSCHEMA_BASE_REMOTE_URL = getattr(settings, 'IMPORT_JSON_SCHEMAS_URI')
 
 logger = DaisyLogger(__name__)
 
 
 class DatasetsExporter:
-    def __init__(self, objects=None):
+    def __init__(
+            self, 
+            include_unpublished=False,
+            objects=None):
+        self.include_unpublished = include_unpublished
         """
         objects would be Django object manager containing datasets to export,
         i.e.:
@@ -46,22 +53,26 @@ class DatasetsExporter:
         for dataset in objects:
             dataset_repr = str(dataset)
             logger.debug(f' * Exporting dataset: "{dataset_repr}"...')
-            try:
-                pd = dataset.to_dict()
-                pd["source"] = settings.SERVER_URL
-                dataset_dicts.append(pd)
-            except Exception as e:
-                logger.error(f'Export failed for dataset {dataset.title}')
-                logger.error(str(e))
-                if verbose:
-                    import traceback
-                    ex = traceback.format_exception(*sys.exc_info())
-                    logger.error('\n'.join([e for e in ex]))
-                if stop_on_error:
-                    raise e
-            logger.debug("   ... complete!")
+            if dataset.is_published or self.include_unpublished:
+                try:
+                    pd = dataset.to_dict()
+                    pd["source"] = settings.SERVER_URL
+                    dataset_dicts.append(pd)
+                except Exception as e:
+                    logger.error(f'Export failed for dataset {dataset.title}')
+                    logger.error(str(e))
+                    if verbose:
+                        import traceback
+                        ex = traceback.format_exception(*sys.exc_info())
+                        logger.error('\n'.join([e for e in ex]))
+                    if stop_on_error:
+                        raise e
+                logger.debug("   ... complete!")
+            else:
+                logger.debug(f' "{dataset_repr}" is not published, it can not be exported')
+
         json.dump({
-            "$schema": "https://raw.githubusercontent.com/elixir-luxembourg/json-schemas/master/schemas/elu-dataset.json",
+            "$schema": urljoin(JSONSCHEMA_BASE_REMOTE_URL, 'elu-dataset.json'),
             "items": dataset_dicts}, buffer, indent=4)
         return buffer
 

core/importer/partners_exporter.py

@@ -5,11 +5,18 @@ from core.models import Partner
 from core.utils import DaisyLogger
 from django.conf import settings
 from io import StringIO
+from urllib.parse import urljoin
+
+JSONSCHEMA_BASE_REMOTE_URL = getattr(settings, 'IMPORT_JSON_SCHEMAS_URI')
 
 logger = DaisyLogger(__name__)
 
 
 class PartnersExporter:
+    def __init__(
+            self,
+            include_unpublished=False):
+        self.include_unpublished = include_unpublished
 
     def export_to_file(self, file_handle, stop_on_error=False, verbose=False):
         result = True
@@ -31,23 +38,26 @@ class PartnersExporter:
             partners = Partner.objects.all()
             for partner in partners:
                 logger.debug(f' * Exporting partner: "{partner.name}"...')
-                try:
-                    pd = partner.to_dict()
-                    pd["source"] = settings.SERVER_URL
-                    partner_dicts.append(pd)
-
-                except Exception as e:
-                    logger.error('Export failed')
-                    logger.error(str(e))
-                    if verbose:
-                        import traceback
-                        ex = traceback.format_exception(*sys.exc_info())
-                        logger.error('\n'.join([e for e in ex]))
-                    if stop_on_error:
-                        raise e
-                logger.debug("   ... complete!")
+                if partner.is_published or self.include_unpublished:
+                    try:
+                        pd = partner.to_dict()
+                        pd["source"] = settings.SERVER_URL
+                        partner_dicts.append(pd)
+
+                    except Exception as e:
+                        logger.error('Export failed')
+                        logger.error(str(e))
+                        if verbose:
+                            import traceback
+                            ex = traceback.format_exception(*sys.exc_info())
+                            logger.error('\n'.join([e for e in ex]))
+                        if stop_on_error:
+                            raise e
+                    logger.debug("   ... complete!")
+                else:
+                    logger.debug(f' "{partner.name}" is not published, it can not be exported')
             json.dump({
-                "$schema": "https://raw.githubusercontent.com/elixir-luxembourg/json-schemas/master/schemas/elu-institution.json",
+                "$schema": urljoin(JSONSCHEMA_BASE_REMOTE_URL, 'elu-institution.json'),
                 "items": partner_dicts}, buffer , indent=4)
             return buffer
 

core/importer/projects_exporter.py

@@ -5,13 +5,20 @@ from core.models import Project
 from core.utils import DaisyLogger
 from django.conf import settings
 from io import StringIO
+from urllib.parse import urljoin
+
+JSONSCHEMA_BASE_REMOTE_URL = getattr(settings, 'IMPORT_JSON_SCHEMAS_URI')
 
 
 logger = DaisyLogger(__name__)
 
 
 class ProjectsExporter:
-    def __init__(self, objects=None):
+    def __init__(
+            self, 
+            include_unpublished=False,
+            objects=None):
+        self.include_unpublished = include_unpublished   
         """
         objects would be Django obejct manager containing projects to export,
         i.e.:
@@ -49,23 +56,26 @@ class ProjectsExporter:
 
         for project in objects:
             logger.debug(f' * Exporting project: "{project.acronym}"...')
-            try:
-                pd = project.to_dict()
-                pd["source"] = settings.SERVER_URL
-                project_dicts.append(pd)
-            except Exception as e:
-                project_repr = str(project)
-                logger.error(f'Export failed for project f{project_repr}')
-                logger.error(str(e))
-                if verbose:
-                    import traceback
-                    ex = traceback.format_exception(*sys.exc_info())
-                    logger.error('\n'.join([e for e in ex]))
-                if stop_on_error:
-                    raise e
-            logger.debug("   ... complete!")
+            if project.is_published or self.include_unpublished:
+                try:
+                    pd = project.to_dict()
+                    pd["source"] = settings.SERVER_URL
+                    project_dicts.append(pd)
+                except Exception as e:
+                    project_repr = str(project)
+                    logger.error(f'Export failed for project f{project_repr}')
+                    logger.error(str(e))
+                    if verbose:
+                        import traceback
+                        ex = traceback.format_exception(*sys.exc_info())
+                        logger.error('\n'.join([e for e in ex]))
+                    if stop_on_error:
+                        raise e
+                logger.debug("   ... complete!")
+            else:
+                logger.debug(f' "{project.acronym}" is not published, it can not be exported')
         json.dump({
-            "$schema": "https://raw.githubusercontent.com/elixir-luxembourg/json-schemas/master/schemas/elu-project.json",
+            "$schema": urljoin(JSONSCHEMA_BASE_REMOTE_URL, 'project.json'),
             "items": project_dicts}, buffer , indent=4)
         return buffer
 

core/management/commands/_private.py

@@ -49,6 +49,7 @@ class ImportBaseCommand(BaseCommand):
             help='JSON file(s) content is validated against JSON schema before import by default.',
             dest='skip_validation',
         )
+        
 
     def handle(self, *args, **options):
         try:
@@ -59,16 +60,16 @@ class ImportBaseCommand(BaseCommand):
             validate = not(options.get('no_validation'))
             path_to_json_directory = options.get('directory')
             skip_on_exist = options.get('skip_on_exist')
+
             importer = self.get_importer(
                                 publish_on_import = publish_on_import,
                                 exit_on_error = exit_on_error, 
                                 verbose = verbose, 
                                 validate = validate,
-                                skip_on_exist = skip_on_exist
-                            )
-
+                                skip_on_exist = skip_on_exist )
             if not(path_to_json_directory or path_to_json_file):
                 raise CommandError('Either directory (--directory) or file (--file) argument must be specified!')
+            
 
             # Import files from directory
             if path_to_json_directory:
@@ -120,16 +121,28 @@ class ExportBaseCommand(BaseCommand):
             default=False
         )
 
+        parser.add_argument(
+            '--include-unpublished',
+            action='store_true',
+            help='Exporter allows export of all records include unpublished ones.',
+            dest='include_unpublished',
+        )
+
     def handle(self, *args, **options):
         if not(options.get('file')):
             raise CommandError('File (--file) argument must be specified!')
 
         try:
+            include_unpublished = options.get('include_unpublished')
             path_to_json_file = options.get('file')
+
+        
+
             with open(path_to_json_file,  mode="w+", encoding='utf-8') as json_file:
-                exp = self.get_exporter()
+                exp = self.get_exporter( include_unpublished = include_unpublished)
                 exp.export_to_file(json_file)
                 self.stdout.write(self.style.SUCCESS("Export complete!"))
+        
 
         except Exception as e:
             msg = f"Something went wrong during the export ({__file__}:class {self.__class__.__name__})! Details:"
@@ -137,5 +150,8 @@ class ExportBaseCommand(BaseCommand):
                 self.style.ERROR(msg))
             self.stderr.write(self.style.ERROR(str(e)))
 
-    def get_exporter(self):
+    def get_exporter(
+            self,
+            include_unpublished=False
+        ):
         raise NotImplementedError("Abstract method: Implement this method in the child class.")

core/management/commands/export_datasets.py

@@ -5,5 +5,8 @@ from core.importer.datasets_exporter import DatasetsExporter
 class Command(ExportBaseCommand):
     help = 'export dataset records to a designated file'
 
-    def get_exporter(self):
-        return DatasetsExporter()
+    def get_exporter(
+            self,
+            include_unpublished=False
+        ):
+        return DatasetsExporter(include_unpublished)

core/management/commands/export_partners.py

@@ -6,5 +6,9 @@ from core.importer.partners_exporter import PartnersExporter
 class Command(ExportBaseCommand):
     help = 'Export partner records to a designated file'
 
-    def get_exporter(self):
-        return PartnersExporter()
+    def get_exporter(
+        self,
+        include_unpublished=False
+    
+    ):
+        return PartnersExporter(include_unpublished)

core/management/commands/export_projects.py

@@ -5,5 +5,8 @@ from core.importer.projects_exporter import ProjectsExporter
 class Command(ExportBaseCommand):
     help = 'export project records to a designated file'
 
-    def get_exporter(self):
-        return ProjectsExporter()
+    def get_exporter(
+            self,
+            include_unpublished=False
+        ):
+        return ProjectsExporter(include_unpublished)

core/tests/importer/test_export.py

@@ -8,6 +8,11 @@ from core.importer.projects_exporter import ProjectsExporter
 from core.importer.JSONSchemaValidator import ProjectJSONSchemaValidator, DatasetJSONSchemaValidator,  InstitutionJSONSchemaValidator
 from test import factories
 
+def export_entities(exporter):
+    buffer = exporter.export_to_buffer(StringIO())
+    dict = json.loads(buffer.getvalue())
+    dict_items = dict['items']
+    return dict_items
 
 @pytest.mark.django_db
 def test_export_projects(celery_session_worker, contact_types, partners, gdpr_roles, storage_resources, can_defer_constraint_checks):
@@ -20,13 +25,14 @@ def test_export_projects(celery_session_worker, contact_types, partners, gdpr_ro
     a_project =  factories.ProjectFactory.create(acronym='Test_PRJ', title='Title of test project.', local_custodians=[rebecca, embury])
     another_project =  factories.ProjectFactory.create(acronym='Another PRJ', title='Title of another test project.', local_custodians=[rebecca, embury])
 
-    exp = ProjectsExporter()
-    buffer = exp.export_to_buffer(StringIO())
+    exp = ProjectsExporter(include_unpublished=False)
+    project_dicts = export_entities(exp)
+    assert 0 == len(project_dicts)
 
-    dict = json.loads(buffer.getvalue())
-    project_dicts = dict['items']
-    assert 2 == len(project_dicts)
+    exp = ProjectsExporter(include_unpublished=True)
+    project_dicts = export_entities(exp)
 
+    assert 2 == len(project_dicts)
     assert "Title of test project." ==  project_dicts[0]['name']
     assert 2 == len(project_dicts[0]['contacts'])
 
@@ -39,13 +45,15 @@ def test_export_projects(celery_session_worker, contact_types, partners, gdpr_ro
 @pytest.mark.django_db
 def test_export_partners(celery_session_worker, contact_types, partners, gdpr_roles, storage_resources, can_defer_constraint_checks):
 
-
-    exp = PartnersExporter()
-    buffer = exp.export_to_buffer(StringIO())
-
-    dict = json.loads(buffer.getvalue())
-    partner_dicts = dict['items']
+    a_partner =  factories.PartnerFactory.create()
+    
+    exp = PartnersExporter(include_unpublished=False)
+    partner_dicts = export_entities(exp)
     assert 96 == len(partner_dicts) # initial data has 96 partners
+    
+    exp = PartnersExporter(include_unpublished=True)
+    partner_dicts = export_entities(exp)
+    assert 97 == len(partner_dicts) # initial data has 96 partners
 
     #TODO add check of more fields
 
@@ -65,11 +73,13 @@ def test_export_datasets(celery_session_worker, contact_types, partners, gdpr_ro
     a_dataset =  factories.DatasetFactory.create(title='A test dataset', project=a_project, local_custodians=[rebecca, embury])
 
 
-    exp = DatasetsExporter()
-    buffer = exp.export_to_buffer(StringIO())
-
-    dict = json.loads(buffer.getvalue())
-    dataset_dicts = dict['items']
+    exp = DatasetsExporter(include_unpublished=False)
+    dataset_dicts = export_entities(exp)
+    assert 0 == len(dataset_dicts)
+    
+    exp = DatasetsExporter(include_unpublished=True)
+    dataset_dicts = export_entities(exp)
+    
     assert 1 == len(dataset_dicts)
     assert "A test dataset" == dataset_dicts[0]['name']
     assert "Test_PRJ" == dataset_dicts[0]['project']