Commit 9ba698a7 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

forgot the files...

parent be8f7095
"""Provides CategoricalETL for PIC-SURE API."""
import json
from typing import List
import pandas as pd
from io import StringIO
from fractalis.data.etl import ETL
from fractalis.data.etls.picsure import shared
class CategoricalETL(ETL):
"""CategoricalETL implements support for PIC-SURE 'categorical' type."""
name = 'pic-sure_categorical_etl'
produces = 'categorical'
@staticmethod
def can_handle(handler: str, descriptor: dict) -> bool:
return handler == 'pic-sure' and \
descriptor['dataType'] == 'categorical'
def extract(self, server: str, token: str, descriptor: dict) -> List[str]:
result_id = shared.submit_query(
query=json.dumps(descriptor['query']), server=server, token=token)
shared.wait_for_completion(
result_id=result_id, server=server, token=token)
raw_data = shared.get_data(
result_id=result_id, server=server, token=token)
return raw_data
def transform(self, raw_data: List[dict], descriptor: dict) -> pd.DataFrame:
df = pd.read_csv(StringIO(raw_data))
feature = df.columns[1]
df.columns = ['id', 'value']
df.insert(1, 'feature', feature)
df[['id']] = df[['id']].astype(str)
df[['value']] = df[['value']].astype(str)
return df
"""Provides NumericalETL for PIC-SURE API."""
import json
from typing import List
import pandas as pd
from io import StringIO
from fractalis.data.etl import ETL
from fractalis.data.etls.picsure import shared
class NumericalETL(ETL):
"""NumericalETL implements support for PIC-SURE 'numerical' type."""
name = 'pic-sure_numerical_etl'
produces = 'numerical'
@staticmethod
def can_handle(handler: str, descriptor: dict) -> bool:
return handler == 'pic-sure' and \
descriptor['dataType'] == 'numerical'
def extract(self, server: str, token: str, descriptor: dict) -> List[str]:
result_id = shared.submit_query(
query=json.dumps(descriptor['query']), server=server, token=token)
shared.wait_for_completion(
result_id=result_id, server=server, token=token)
raw_data = shared.get_data(
result_id=result_id, server=server, token=token)
return raw_data
def transform(self, raw_data: List[dict], descriptor: dict) -> pd.DataFrame:
df = pd.read_csv(StringIO(raw_data))
feature = df.columns[1]
df.columns = ['id', 'value']
df.insert(1, 'feature', feature)
df[['id']] = df[['id']].astype(str)
df[['value']] = df[['value']].astype(float)
return df
"""This module provides PicSureHandler,
an implementation of ETLHandler for PIC-SURE."""
import logging
from fractalis.data.etlhandler import ETLHandler
logger = logging.getLogger(__name__)
class PicSureHandler(ETLHandler):
"""This ETLHandler provides integration with PIC-SURE."""
_handler = 'pic-sure'
@staticmethod
def make_label(descriptor: dict) -> str:
return descriptor['query']['select'][0]['alias']
def _get_token_for_credentials(self, server: str, auth: dict) -> str:
return auth['token']
from time import sleep
import requests
def submit_query(query: str, server: str, token: str) -> int:
r = requests.post(
url='{}/rest/v1/queryService/runQuery'.format(server),
data=query,
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(token)
}
)
r.raise_for_status()
result_id = r.json()['resultId']
return result_id
def wait_for_completion(result_id: int, server, token):
def _check_status():
return requests.get(
url='{}/rest/v1/resultService/resultStatus/{}'.format(
server, result_id),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(token)
}
).json()
while _check_status()['status'] == 'RUNNING':
sleep(1)
def get_data(result_id, server, token):
r = requests.get(
url='{}/rest/v1/resultService/result/{}/CSV'.format(
server, result_id),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(token)
}
)
r.raise_for_status()
return r.text
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment