Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Pinar Alper
metadata-tools
Commits
d1b10b70
Commit
d1b10b70
authored
Jan 19, 2021
by
Jacek Lebioda
Browse files
feat: use new format for export, added datetime to log entries and hyphen in use restrictions
parent
ac7535c6
Changes
4
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
d1b10b70
...
...
@@ -3,3 +3,6 @@ __pycache__/
# Visual studio code
.vscode
project_venv
env
*.egg-info
*.log
metadata_tools/importxls/dish_xls_exporter.py
View file @
d1b10b70
...
...
@@ -8,7 +8,10 @@ from .export_utils import get_partners_from_daisy, process_possible_date, proces
class
DishXlsExporter
:
def
__init__
(
self
):
logging
.
basicConfig
(
filename
=
'export_dishxls.log'
,
level
=
logging
.
DEBUG
)
logging
.
basicConfig
(
filename
=
'export_dishxls.log'
,
level
=
logging
.
DEBUG
,
format
=
'%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s'
,
datefmt
=
'%Y-%m-%d %H:%M:%S'
)
institutions
=
get_partners_from_daisy
()
self
.
inst_dict
=
{}
self
.
inst_ac_dict
=
{}
...
...
@@ -145,19 +148,19 @@ class DishXlsExporter:
use_restrictions
=
[]
if
process_yes_no_dontknow_answer
(
sheet
[
24
,
1
]):
use_restrictions
.
append
({
'use_class'
:
'RS[XX]'
,
use_restrictions
.
append
({
'use_class'
:
'RS
-
[XX]'
,
'use_restriction_rule'
:
"CONSTRAINTS"
,
'use_class_note'
:
sheet
[
25
,
1
]})
elif
process_yes_no_dontknow_answer
(
sheet
[
24
,
1
])
is
not
None
:
use_restrictions
.
append
({
'use_class'
:
'RS[XX]'
,
use_restrictions
.
append
({
'use_class'
:
'RS
-
[XX]'
,
'use_restriction_rule'
:
"NO_CONSTRAINTS"
,
'use_class_note'
:
sheet
[
25
,
1
]})
if
process_yes_no_dontknow_answer
(
sheet
[
26
,
1
]):
use_restrictions
.
append
({
'use_class'
:
'GS[XX]'
,
use_restrictions
.
append
({
'use_class'
:
'GS
-
[XX]'
,
'use_restriction_rule'
:
"CONSTRAINTS"
,
'use_class_note'
:
sheet
[
27
,
1
]})
elif
process_yes_no_dontknow_answer
(
sheet
[
26
,
1
])
is
not
None
:
use_restrictions
.
append
({
'use_class'
:
'GS[XX]'
,
use_restrictions
.
append
({
'use_class'
:
'GS
-
[XX]'
,
'use_restriction_rule'
:
"NO_CONSTRAINTS"
,
'use_class_note'
:
sheet
[
27
,
1
]})
...
...
metadata_tools/importxls/export_utils.py
View file @
d1b10b70
from
datetime
import
datetime
import
datetime
as
dt
import
json
import
logging
import
urllib
from
datetime
import
datetime
from
json
import
dumps
from
urllib.error
import
HTTPError
,
URLError
from
socket
import
timeout
import
logging
from
urllib.error
import
HTTPError
,
URLError
def
process_yes_no_answer
(
answer
):
"""
...
...
@@ -107,7 +109,6 @@ def get_lines_from_string(a_string):
def
get_partners_from_daisy
():
entities_json_str
=
None
try
:
urlEntities
=
urllib
.
parse
.
urljoin
(
'https://daisy.lcsb.uni.lu/api/'
,
'partners'
)
...
...
@@ -130,3 +131,11 @@ def get_partners_from_daisy():
entities_json_str
=
dumps
(
'{}'
)
return
json
.
loads
(
entities_json_str
)
def
save_exported_datasets_to_file
(
exported_datasets
,
output_file
):
obj
=
{
"$schema"
:
"https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/elu-dataset.json"
,
"items"
:
exported_datasets
}
return
json
.
dump
(
obj
,
output_file
,
indent
=
4
)
tests/importxls/test_dish_xls_exporter.py
View file @
d1b10b70
from
unittest
import
TestCase
import
os
from
metadata_tools.importxls.dish_xls_exporter
import
DishXlsExporter
import
json
import
os
from
pathlib
import
Path
from
unittest
import
TestCase
class
TestDishXlsPartnerExtractor
(
TestCase
):
from
metadata_tools.importxls.export_utils
import
save_exported_datasets_to_file
from
metadata_tools.importxls.dish_xls_exporter
import
DishXlsExporter
def
test_dish_export
(
self
):
class
TestDishXlsPartnerExtractor
(
TestCase
):
def
test_dish_export
(
self
):
exporter
=
DishXlsExporter
()
#full_file_path = os.path.join(os.path.dirname(__file__), 'resources', 'UL_Extended_Data_Information_Sheet-8.xlsx')
...
...
@@ -18,7 +20,7 @@ class TestDishXlsPartnerExtractor(TestCase):
full_file_path
=
os
.
path
.
join
(
dirName
,
fname
)
dataset_dict
=
exporter
.
export_submission
(
full_file_path
)
with
open
(
Path
(
full_file_path
).
stem
+
".json"
,
'w'
)
as
outfile
:
json
.
dump
([
dataset_dict
]
,
outfile
,
indent
=
4
)
save_exported_datasets_to_file
(
dataset_dict
,
outfile
)
except
ValueError
:
print
(
"Could not import {}"
.
format
(
full_file_path
))
return
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment