Gitlab is now using https://gitlab.lcsb.uni.lu as it's primary address. Please update your bookmarks. FAQ.

Commit 137db9fb authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

data frames are now pickled instead of csv to conserve types

parent 33ae1a38
......@@ -34,8 +34,8 @@
const fjs = fractal.init({
handler: 'demo',
dataSource: 'https://' + window.location.host,
fractalisNode: 'http://' + window.location.host,
dataSource: location.protocol + '//' + window.location.host,
fractalisNode: location.protocol + '//' + window.location.host,
getAuth () {
return {token: ''}
},
......
......@@ -7,7 +7,7 @@ import logging
from uuid import UUID
from typing import List, Tuple, Union
from pandas import read_csv, DataFrame
from pandas import read_pickle, DataFrame
from celery import Task
from Cryptodome.Cipher import AES
......@@ -99,7 +99,7 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
if decrypt:
return self.secure_load(file_path)
else:
df = read_csv(file_path)
df = read_pickle(file_path, compression='gzip')
return df
@staticmethod
......
......@@ -152,7 +152,7 @@ class ETL(Task, metaclass=abc.ABCMeta):
:param file_path: File to write to.
"""
os.makedirs(os.path.dirname(file_path), exist_ok=True)
data_frame.to_csv(file_path, index=False)
data_frame.to_pickle(file_path, compression='gzip')
def run(self, server: str, token: str,
descriptor: dict, file_path: str,
......
......@@ -11,7 +11,7 @@ class CategoricalETL(ETL):
"""CategoricalETL implements support for the
categorical data of the wine quality data set. """
name = 'demo-wine-quality-etl'
name = 'demo-wine-quality-categorical-etl'
produces = 'categorical'
@staticmethod
......
......@@ -11,7 +11,7 @@ class NumericalETL(ETL):
"""NumericalETL implements support for the
numerical data of the wine quality data set. """
name = 'demo-wine-quality-etl'
name = 'demo-wine-quality-numerical-etl'
produces = 'numerical'
@staticmethod
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment