Commit 137db9fb authored by Sascha Herzinger's avatar Sascha Herzinger

data frames are now pickled instead of csv to conserve types

parent 33ae1a38
......@@ -34,8 +34,8 @@
const fjs = fractal.init({
handler: 'demo',
dataSource: 'https://' + window.location.host,
fractalisNode: 'http://' + window.location.host,
dataSource: location.protocol + '//' + window.location.host,
fractalisNode: location.protocol + '//' + window.location.host,
getAuth () {
return {token: ''}
},
......
......@@ -7,7 +7,7 @@ import logging
from uuid import UUID
from typing import List, Tuple, Union
from pandas import read_csv, DataFrame
from pandas import read_pickle, DataFrame
from celery import Task
from Cryptodome.Cipher import AES
......@@ -99,7 +99,7 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
if decrypt:
return self.secure_load(file_path)
else:
df = read_csv(file_path)
df = read_pickle(file_path, compression='gzip')
return df
@staticmethod
......
......@@ -152,7 +152,7 @@ class ETL(Task, metaclass=abc.ABCMeta):
:param file_path: File to write to.
"""
os.makedirs(os.path.dirname(file_path), exist_ok=True)
data_frame.to_csv(file_path, index=False)
data_frame.to_pickle(file_path, compression='gzip')
def run(self, server: str, token: str,
descriptor: dict, file_path: str,
......
......@@ -11,7 +11,7 @@ class CategoricalETL(ETL):
"""CategoricalETL implements support for the
categorical data of the wine quality data set. """
name = 'demo-wine-quality-etl'
name = 'demo-wine-quality-categorical-etl'
produces = 'categorical'
@staticmethod
......
......@@ -11,7 +11,7 @@ class NumericalETL(ETL):
"""NumericalETL implements support for the
numerical data of the wine quality data set. """
name = 'demo-wine-quality-etl'
name = 'demo-wine-quality-numerical-etl'
produces = 'numerical'
@staticmethod
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment