Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Fractalis
fractalis
Commits
137db9fb
Commit
137db9fb
authored
May 04, 2018
by
Sascha Herzinger
Browse files
data frames are now pickled instead of csv to conserve types
parent
33ae1a38
Changes
5
Hide whitespace changes
Inline
Side-by-side
docker/config/nginx/html/demo.html
View file @
137db9fb
...
...
@@ -34,8 +34,8 @@
const
fjs
=
fractal
.
init
({
handler
:
'
demo
'
,
dataSource
:
'
https:
//
'
+
window
.
location
.
host
,
fractalisNode
:
'
http:
//
'
+
window
.
location
.
host
,
dataSource
:
location
.
protocol
+
'
//
'
+
window
.
location
.
host
,
fractalisNode
:
location
.
protocol
+
'
//
'
+
window
.
location
.
host
,
getAuth
()
{
return
{
token
:
''
}
},
...
...
fractalis/analytics/task.py
View file @
137db9fb
...
...
@@ -7,7 +7,7 @@ import logging
from
uuid
import
UUID
from
typing
import
List
,
Tuple
,
Union
from
pandas
import
read_
csv
,
DataFrame
from
pandas
import
read_
pickle
,
DataFrame
from
celery
import
Task
from
Cryptodome.Cipher
import
AES
...
...
@@ -99,7 +99,7 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
if
decrypt
:
return
self
.
secure_load
(
file_path
)
else
:
df
=
read_
csv
(
file_path
)
df
=
read_
pickle
(
file_path
,
compression
=
'gzip'
)
return
df
@
staticmethod
...
...
fractalis/data/etl.py
View file @
137db9fb
...
...
@@ -152,7 +152,7 @@ class ETL(Task, metaclass=abc.ABCMeta):
:param file_path: File to write to.
"""
os
.
makedirs
(
os
.
path
.
dirname
(
file_path
),
exist_ok
=
True
)
data_frame
.
to_
csv
(
file_path
,
index
=
False
)
data_frame
.
to_
pickle
(
file_path
,
compression
=
'gzip'
)
def
run
(
self
,
server
:
str
,
token
:
str
,
descriptor
:
dict
,
file_path
:
str
,
...
...
fractalis/data/etls/demo/demo_etl_categorical.py
View file @
137db9fb
...
...
@@ -11,7 +11,7 @@ class CategoricalETL(ETL):
"""CategoricalETL implements support for the
categorical data of the wine quality data set. """
name
=
'demo-wine-quality-etl'
name
=
'demo-wine-quality-
categorical-
etl'
produces
=
'categorical'
@
staticmethod
...
...
fractalis/data/etls/demo/demo_etl_numerical.py
View file @
137db9fb
...
...
@@ -11,7 +11,7 @@ class NumericalETL(ETL):
"""NumericalETL implements support for the
numerical data of the wine quality data set. """
name
=
'demo-wine-quality-etl'
name
=
'demo-wine-quality-
numerical-
etl'
produces
=
'numerical'
@
staticmethod
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment