Commits (6)
......@@ -5,12 +5,19 @@ from core.models import Dataset
from core.utils import DaisyLogger
from django.conf import settings
from io import StringIO
from urllib.parse import urljoin
JSONSCHEMA_BASE_REMOTE_URL = getattr(settings, 'IMPORT_JSON_SCHEMAS_URI')
logger = DaisyLogger(__name__)
class DatasetsExporter:
def __init__(self, objects=None):
def __init__(
self,
include_unpublished=False,
objects=None):
self.include_unpublished = include_unpublished
"""
objects would be Django object manager containing datasets to export,
i.e.:
......@@ -46,22 +53,26 @@ class DatasetsExporter:
for dataset in objects:
dataset_repr = str(dataset)
logger.debug(f' * Exporting dataset: "{dataset_repr}"...')
try:
pd = dataset.to_dict()
pd["source"] = settings.SERVER_URL
dataset_dicts.append(pd)
except Exception as e:
logger.error(f'Export failed for dataset {dataset.title}')
logger.error(str(e))
if verbose:
import traceback
ex = traceback.format_exception(*sys.exc_info())
logger.error('\n'.join([e for e in ex]))
if stop_on_error:
raise e
logger.debug(" ... complete!")
if dataset.is_published or self.include_unpublished:
try:
pd = dataset.to_dict()
pd["source"] = settings.SERVER_URL
dataset_dicts.append(pd)
except Exception as e:
logger.error(f'Export failed for dataset {dataset.title}')
logger.error(str(e))
if verbose:
import traceback
ex = traceback.format_exception(*sys.exc_info())
logger.error('\n'.join([e for e in ex]))
if stop_on_error:
raise e
logger.debug(" ... complete!")
else:
logger.debug(f' "{dataset_repr}" is not published, it can not be exported')
json.dump({
"$schema": "https://raw.githubusercontent.com/elixir-luxembourg/json-schemas/master/schemas/elu-dataset.json",
"$schema": urljoin(JSONSCHEMA_BASE_REMOTE_URL, 'elu-dataset.json'),
"items": dataset_dicts}, buffer, indent=4)
return buffer
......@@ -5,11 +5,18 @@ from core.models import Partner
from core.utils import DaisyLogger
from django.conf import settings
from io import StringIO
from urllib.parse import urljoin
JSONSCHEMA_BASE_REMOTE_URL = getattr(settings, 'IMPORT_JSON_SCHEMAS_URI')
logger = DaisyLogger(__name__)
class PartnersExporter:
def __init__(
self,
include_unpublished=False):
self.include_unpublished = include_unpublished
def export_to_file(self, file_handle, stop_on_error=False, verbose=False):
result = True
......@@ -31,23 +38,26 @@ class PartnersExporter:
partners = Partner.objects.all()
for partner in partners:
logger.debug(f' * Exporting partner: "{partner.name}"...')
try:
pd = partner.to_dict()
pd["source"] = settings.SERVER_URL
partner_dicts.append(pd)
except Exception as e:
logger.error('Export failed')
logger.error(str(e))
if verbose:
import traceback
ex = traceback.format_exception(*sys.exc_info())
logger.error('\n'.join([e for e in ex]))
if stop_on_error:
raise e
logger.debug(" ... complete!")
if partner.is_published or self.include_unpublished:
try:
pd = partner.to_dict()
pd["source"] = settings.SERVER_URL
partner_dicts.append(pd)
except Exception as e:
logger.error('Export failed')
logger.error(str(e))
if verbose:
import traceback
ex = traceback.format_exception(*sys.exc_info())
logger.error('\n'.join([e for e in ex]))
if stop_on_error:
raise e
logger.debug(" ... complete!")
else:
logger.debug(f' "{partner.name}" is not published, it can not be exported')
json.dump({
"$schema": "https://raw.githubusercontent.com/elixir-luxembourg/json-schemas/master/schemas/elu-institution.json",
"$schema": urljoin(JSONSCHEMA_BASE_REMOTE_URL, 'elu-institution.json'),
"items": partner_dicts}, buffer , indent=4)
return buffer
......
......@@ -5,13 +5,20 @@ from core.models import Project
from core.utils import DaisyLogger
from django.conf import settings
from io import StringIO
from urllib.parse import urljoin
JSONSCHEMA_BASE_REMOTE_URL = getattr(settings, 'IMPORT_JSON_SCHEMAS_URI')
logger = DaisyLogger(__name__)
class ProjectsExporter:
def __init__(self, objects=None):
def __init__(
self,
include_unpublished=False,
objects=None):
self.include_unpublished = include_unpublished
"""
objects would be Django obejct manager containing projects to export,
i.e.:
......@@ -49,23 +56,26 @@ class ProjectsExporter:
for project in objects:
logger.debug(f' * Exporting project: "{project.acronym}"...')
try:
pd = project.to_dict()
pd["source"] = settings.SERVER_URL
project_dicts.append(pd)
except Exception as e:
project_repr = str(project)
logger.error(f'Export failed for project f{project_repr}')
logger.error(str(e))
if verbose:
import traceback
ex = traceback.format_exception(*sys.exc_info())
logger.error('\n'.join([e for e in ex]))
if stop_on_error:
raise e
logger.debug(" ... complete!")
if project.is_published or self.include_unpublished:
try:
pd = project.to_dict()
pd["source"] = settings.SERVER_URL
project_dicts.append(pd)
except Exception as e:
project_repr = str(project)
logger.error(f'Export failed for project f{project_repr}')
logger.error(str(e))
if verbose:
import traceback
ex = traceback.format_exception(*sys.exc_info())
logger.error('\n'.join([e for e in ex]))
if stop_on_error:
raise e
logger.debug(" ... complete!")
else:
logger.debug(f' "{project.acronym}" is not published, it can not be exported')
json.dump({
"$schema": "https://raw.githubusercontent.com/elixir-luxembourg/json-schemas/master/schemas/elu-project.json",
"$schema": urljoin(JSONSCHEMA_BASE_REMOTE_URL, 'project.json'),
"items": project_dicts}, buffer , indent=4)
return buffer
......
......@@ -49,6 +49,7 @@ class ImportBaseCommand(BaseCommand):
help='JSON file(s) content is validated against JSON schema before import by default.',
dest='skip_validation',
)
def handle(self, *args, **options):
try:
......@@ -59,16 +60,16 @@ class ImportBaseCommand(BaseCommand):
validate = not(options.get('no_validation'))
path_to_json_directory = options.get('directory')
skip_on_exist = options.get('skip_on_exist')
importer = self.get_importer(
publish_on_import = publish_on_import,
exit_on_error = exit_on_error,
verbose = verbose,
validate = validate,
skip_on_exist = skip_on_exist
)
skip_on_exist = skip_on_exist )
if not(path_to_json_directory or path_to_json_file):
raise CommandError('Either directory (--directory) or file (--file) argument must be specified!')
# Import files from directory
if path_to_json_directory:
......@@ -120,16 +121,28 @@ class ExportBaseCommand(BaseCommand):
default=False
)
parser.add_argument(
'--include-unpublished',
action='store_true',
help='Exporter allows export of all records include unpublished ones.',
dest='include_unpublished',
)
def handle(self, *args, **options):
if not(options.get('file')):
raise CommandError('File (--file) argument must be specified!')
try:
include_unpublished = options.get('include_unpublished')
path_to_json_file = options.get('file')
with open(path_to_json_file, mode="w+", encoding='utf-8') as json_file:
exp = self.get_exporter()
exp = self.get_exporter( include_unpublished = include_unpublished)
exp.export_to_file(json_file)
self.stdout.write(self.style.SUCCESS("Export complete!"))
except Exception as e:
msg = f"Something went wrong during the export ({__file__}:class {self.__class__.__name__})! Details:"
......@@ -137,5 +150,8 @@ class ExportBaseCommand(BaseCommand):
self.style.ERROR(msg))
self.stderr.write(self.style.ERROR(str(e)))
def get_exporter(self):
def get_exporter(
self,
include_unpublished=False
):
raise NotImplementedError("Abstract method: Implement this method in the child class.")
......@@ -5,5 +5,8 @@ from core.importer.datasets_exporter import DatasetsExporter
class Command(ExportBaseCommand):
help = 'export dataset records to a designated file'
def get_exporter(self):
return DatasetsExporter()
def get_exporter(
self,
include_unpublished=False
):
return DatasetsExporter(include_unpublished)
......@@ -6,5 +6,9 @@ from core.importer.partners_exporter import PartnersExporter
class Command(ExportBaseCommand):
help = 'Export partner records to a designated file'
def get_exporter(self):
return PartnersExporter()
def get_exporter(
self,
include_unpublished=False
):
return PartnersExporter(include_unpublished)
......@@ -5,5 +5,8 @@ from core.importer.projects_exporter import ProjectsExporter
class Command(ExportBaseCommand):
help = 'export project records to a designated file'
def get_exporter(self):
return ProjectsExporter()
def get_exporter(
self,
include_unpublished=False
):
return ProjectsExporter(include_unpublished)
......@@ -8,6 +8,11 @@ from core.importer.projects_exporter import ProjectsExporter
from core.importer.JSONSchemaValidator import ProjectJSONSchemaValidator, DatasetJSONSchemaValidator, InstitutionJSONSchemaValidator
from test import factories
def export_entities(exporter):
buffer = exporter.export_to_buffer(StringIO())
dict = json.loads(buffer.getvalue())
dict_items = dict['items']
return dict_items
@pytest.mark.django_db
def test_export_projects(celery_session_worker, contact_types, partners, gdpr_roles, storage_resources, can_defer_constraint_checks):
......@@ -20,13 +25,14 @@ def test_export_projects(celery_session_worker, contact_types, partners, gdpr_ro
a_project = factories.ProjectFactory.create(acronym='Test_PRJ', title='Title of test project.', local_custodians=[rebecca, embury])
another_project = factories.ProjectFactory.create(acronym='Another PRJ', title='Title of another test project.', local_custodians=[rebecca, embury])
exp = ProjectsExporter()
buffer = exp.export_to_buffer(StringIO())
exp = ProjectsExporter(include_unpublished=False)
project_dicts = export_entities(exp)
assert 0 == len(project_dicts)
dict = json.loads(buffer.getvalue())
project_dicts = dict['items']
assert 2 == len(project_dicts)
exp = ProjectsExporter(include_unpublished=True)
project_dicts = export_entities(exp)
assert 2 == len(project_dicts)
assert "Title of test project." == project_dicts[0]['name']
assert 2 == len(project_dicts[0]['contacts'])
......@@ -39,13 +45,15 @@ def test_export_projects(celery_session_worker, contact_types, partners, gdpr_ro
@pytest.mark.django_db
def test_export_partners(celery_session_worker, contact_types, partners, gdpr_roles, storage_resources, can_defer_constraint_checks):
exp = PartnersExporter()
buffer = exp.export_to_buffer(StringIO())
dict = json.loads(buffer.getvalue())
partner_dicts = dict['items']
a_partner = factories.PartnerFactory.create()
exp = PartnersExporter(include_unpublished=False)
partner_dicts = export_entities(exp)
assert 96 == len(partner_dicts) # initial data has 96 partners
exp = PartnersExporter(include_unpublished=True)
partner_dicts = export_entities(exp)
assert 97 == len(partner_dicts) # initial data has 96 partners
#TODO add check of more fields
......@@ -65,11 +73,13 @@ def test_export_datasets(celery_session_worker, contact_types, partners, gdpr_ro
a_dataset = factories.DatasetFactory.create(title='A test dataset', project=a_project, local_custodians=[rebecca, embury])
exp = DatasetsExporter()
buffer = exp.export_to_buffer(StringIO())
dict = json.loads(buffer.getvalue())
dataset_dicts = dict['items']
exp = DatasetsExporter(include_unpublished=False)
dataset_dicts = export_entities(exp)
assert 0 == len(dataset_dicts)
exp = DatasetsExporter(include_unpublished=True)
dataset_dicts = export_entities(exp)
assert 1 == len(dataset_dicts)
assert "A test dataset" == dataset_dicts[0]['name']
assert "Test_PRJ" == dataset_dicts[0]['project']
......