Commit 1858f8f9 authored by Piotr Gawron's avatar Piotr Gawron
Browse files

overlays are uploaded in chunks (so big files can be uploaded as well)

parent de3a911b
Pipeline #2580 passed with stage
in 40 seconds
......@@ -2,8 +2,33 @@
<project version="4">
<component name="ProjectCodeStyleSettingsManager">
<option name="PER_PROJECT_SETTINGS">
<value />
<value>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</value>
</option>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="custom" />
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</component>
</project>
\ No newline at end of file
......@@ -8,9 +8,9 @@
<excludeFolder url="file://$MODULE_DIR$/dist" />
<excludeFolder url="file://$MODULE_DIR$/target" />
<excludeFolder url="file://$MODULE_DIR$/temp" />
<excludeFolder url="file://$MODULE_DIR$/testFiles" />
<excludeFolder url="file://$MODULE_DIR$/tmp" />
</content>
<content url="file://$MODULE_DIR$/testFiles/apiCalls" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="chai-DefinitelyTyped" level="application" />
......
......@@ -3124,6 +3124,11 @@
"acorn": "4.0.13"
}
},
"text-encoding": {
"version": "0.6.4",
"resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz",
"integrity": "sha1-45mpgiV6J22uQou5KEXLcb3CbRk="
},
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
......
......@@ -47,6 +47,7 @@
"log4js": "0.6.38",
"pileup": "^0.6.8",
"request": "^2.82.0",
"spectrum-colorpicker": "^1.8.0"
"spectrum-colorpicker": "^1.8.0",
"text-encoding": "^0.6.4"
}
}
This diff is collapsed.
......@@ -15,6 +15,7 @@ var logger = require('../logger');
var HttpStatus = require('http-status-codes');
var Promise = require("bluebird");
var TextEncoder = require('text-encoding').TextEncoder;
function AddOverlayDialog(params) {
AbstractGuiElement.call(this, params);
......@@ -36,7 +37,7 @@ AddOverlayDialog.prototype.createGui = function () {
var nameInput = Functions.createElement({
type: "input",
inputType: "text",
name: "overlay-name",
name: "overlay-name"
});
content.appendChild(nameInput);
content.appendChild(guiUtils.createNewLine());
......@@ -45,7 +46,7 @@ AddOverlayDialog.prototype.createGui = function () {
content.appendChild(guiUtils.createNewLine());
var descriptionInput = Functions.createElement({
type: "textarea",
name: "overlay-description",
name: "overlay-description"
});
content.appendChild(descriptionInput);
content.appendChild(guiUtils.createNewLine());
......@@ -54,10 +55,10 @@ AddOverlayDialog.prototype.createGui = function () {
var fileInput = Functions.createElement({
type: "input",
inputType: "file",
name: "overlay-file",
name: "overlay-file"
});
fileInput.addEventListener("change", function () {
return self.processFile(fileInput.files[0]);
return self.processFile(fileInput.files[0]).then(null, GuiConnector.alert);
}, false);
content.appendChild(fileInput);
content.appendChild(guiUtils.createNewLine());
......@@ -66,7 +67,7 @@ AddOverlayDialog.prototype.createGui = function () {
content.appendChild(guiUtils.createNewLine());
var contentInput = Functions.createElement({
type: "textarea",
name: "overlay-content",
name: "overlay-content"
});
content.appendChild(contentInput);
content.appendChild(guiUtils.createNewLine());
......@@ -80,7 +81,7 @@ AddOverlayDialog.prototype.processFile = function (file) {
if (file) {
return new Promise(function (resolve, reject) {
var reader = new FileReader();
reader.readAsText(file, "UTF-8");
reader.readAsArrayBuffer(file);
reader.onload = function (evt) {
try {
var overlayParser = new OverlayParser();
......@@ -119,6 +120,10 @@ AddOverlayDialog.prototype.processFile = function (file) {
};
AddOverlayDialog.prototype.setFileContent = function (fileContent) {
if (typeof fileContent === 'string' || fileContent instanceof String) {
fileContent = new TextEncoder("UTF8").encode(fileContent);
}
this._fileContent = fileContent;
};
......@@ -129,7 +134,7 @@ AddOverlayDialog.prototype.getFileContent = function () {
if (contentInput.value !== undefined && contentInput.value !== null) {
contentInput.value = contentInput.value.trim();
if (contentInput.value !== "") {
self._fileContent = contentInput.value;
self.setFileContent(contentInput.value);
}
}
if (self._fileContent === undefined) {
......@@ -151,13 +156,21 @@ AddOverlayDialog.prototype.addOverlay = function () {
var overlay = new LayoutData({
name: nameInput.value,
description: descriptionInput.value,
content: self.getFileContent(),
filename: filename,
filename: filename
});
if (filename === undefined || filename === "") {
filename = "unknown.txt";
}
GuiConnector.showProcessing();
return ServerConnector.addOverlay({
overlay: overlay,
projectId: self.getProject().getProjectId(),
return ServerConnector.uploadFile({
filename: filename,
content: self.getFileContent()
}).then(function (file) {
return ServerConnector.addOverlay({
fileId: file.id,
overlay: overlay,
projectId: self.getProject().getProjectId()
});
}).then(function (result) {
overlay = result;
GuiConnector.hideProcessing();
......@@ -180,8 +193,6 @@ AddOverlayDialog.prototype.open = function () {
var fileContent = self.getFileContent();
if (fileContent === null) {
GuiConnector.alert("Neither file was selected nor data was entered");
} else if (fileContent.length > 1024 * 256) {
GuiConnector.alert("File to big.<br>Please reduce file size or contact administrators.");
} else {
return self.addOverlay().then(function (result) {
$(dialog).dialog("close");
......@@ -207,7 +218,7 @@ AddOverlayDialog.prototype.open = function () {
$(div).dialog({
title: "Add overlay",
buttons: buttons,
modal: true,
modal: true
});
}
......
......@@ -2,11 +2,15 @@
var logger = require('./../logger');
var LayoutData = require('./data/LayoutData');
var TextDecoder = require('text-encoding').TextDecoder;
function OverlayParser() {
}
OverlayParser.prototype.parse = function (content) {
if (content instanceof Uint8Array || content instanceof ArrayBuffer) {
content = new TextDecoder("UTF8").decode(content);
}
var data = {content: content};
var lines = content.split("\n");
for (var i = 0; i < lines.length; i++) {
......
......@@ -187,7 +187,7 @@ describe('ServerConnector', function () {
description: "test desc",
content: "name color\nCAPN1 #00FF00\nPARK7 #AC0000",
filename: "test.txt"
}),
})
}).then(function (overlay) {
assert.ok(overlay);
});
......
......@@ -3,9 +3,12 @@
require("../mocha-config.js");
var OverlayParser = require('../../../main/js/map/OverlayParser');
var ServerConnector = require('../ServerConnector-mock');
var chai = require('chai');
var assert = chai.assert;
var TextEncoder = require('text-encoding').TextEncoder;
describe('OverlayParser', function () {
describe('parse', function () {
it('simple', function () {
......@@ -18,6 +21,16 @@ describe('OverlayParser', function () {
assert.equal(overlay.getName(), "some Name");
assert.ok(overlay.getContent());
});
it('Uint8Array', function () {
var parser = new OverlayParser();
var fileContent = new TextEncoder("UTF8").encode("#NAME=some Name\n#DESCRIPTION=xxx\nname\tvalue\ns1\t1");
var overlay = parser.parse(fileContent);
assert.ok(overlay);
assert.equal(overlay.getDescription(), "xxx");
assert.equal(overlay.getName(), "some Name");
assert.ok(overlay.getContent());
});
it('with type', function () {
return ServerConnector.sendGetRequest("testFiles/overlay/good.txt").then(function (fileContent) {
......
{
"owner": "admin",
"filename": "unknown.txt",
"uploadedDataLength": 5,
"length": 5,
"id": 6792
}
\ No newline at end of file
{
"owner": "admin",
"filename": "test.txt",
"uploadedDataLength": 0,
"length": 3,
"id": 6792
}
\ No newline at end of file
......@@ -29,139 +29,145 @@ import lcsb.mapviewer.services.view.LayoutView;
@RestController
public class OverlayController extends BaseController {
/**
* Default class logger.
*/
@SuppressWarnings("unused")
private Logger logger = Logger.getLogger(OverlayController.class);
@Autowired
private OverlayRestImpl overlayRestImp;
@RequestMapping(value = "/projects/{projectId}/overlays/", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public List<LayoutView> getOverlayList(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@RequestParam(value = "creator", defaultValue = "") String creator, //
@RequestParam(value = "publicOverlay", defaultValue = "") String publicOverlay //
) throws SecurityException, QueryException {
return overlayRestImp.getOverlayList(token, projectId, creator, publicOverlay);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public LayoutView getOverlayById(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId//
) throws SecurityException, QueryException {
return overlayRestImp.getOverlayById(token, projectId, overlayId);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/models/{modelId}/bioEntities/", method = { RequestMethod.GET },
produces = { MediaType.APPLICATION_JSON_VALUE })
public List<Map<String, Object>> getOverlayElements(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId, @RequestParam(value = "columns", defaultValue = "") String columns)
throws SecurityException, QueryException {
return overlayRestImp.getOverlayElements(token, projectId, Integer.valueOf(overlayId), columns);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/models/{modelId}/bioEntities/reactions/{reactionId}/", method = { RequestMethod.GET },
produces = { MediaType.APPLICATION_JSON_VALUE })
public Map<String, Object> getFullReaction(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "modelId") String modelId, //
@PathVariable(value = "overlayId") String overlayId, //
@PathVariable(value = "reactionId") String reactionId, //
@RequestParam(value = "columns", defaultValue = "") String columns //
) throws SecurityException, QueryException {
return overlayRestImp
.getOverlayElement(token, projectId, Integer.valueOf(modelId), Integer.valueOf(overlayId), Integer.valueOf(reactionId), "REACTION", columns);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/models/{modelId}/bioEntities/elements/{elementId}/", method = { RequestMethod.GET },
produces = { MediaType.APPLICATION_JSON_VALUE })
public Map<String, Object> getFullSpecies(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "modelId") String modelId, //
@PathVariable(value = "overlayId") String overlayId, //
@PathVariable(value = "elementId") String reactionId, //
@RequestParam(value = "columns", defaultValue = "") String columns //
) throws SecurityException, QueryException {
return overlayRestImp
.getOverlayElement(token, projectId, Integer.valueOf(modelId), Integer.valueOf(overlayId), Integer.valueOf(reactionId), "ALIAS", columns);
}
@RequestMapping(value = "/projects/{projectId}/overlays/", method = { RequestMethod.POST }, produces = { MediaType.APPLICATION_JSON_VALUE })
public LayoutView addOverlay(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@RequestParam(value = "name") String name, //
@RequestParam(value = "description") String description, //
@RequestParam(value = "content") String content, //
@RequestParam(value = "filename") String filename, //
@RequestParam(value = "type", defaultValue = "") String type //
) throws SecurityException, QueryException, IOException {
return overlayRestImp.addOverlay(token, projectId, name, description, content, filename, type);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}", method = { RequestMethod.DELETE }, produces = { MediaType.APPLICATION_JSON_VALUE })
public Map<String, Object> removeOverlay(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId //
) throws SecurityException, QueryException, IOException {
return overlayRestImp.removeOverlay(token, projectId, overlayId);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}", method = { RequestMethod.PATCH }, produces = { MediaType.APPLICATION_JSON_VALUE })
public LayoutView updateOverlay(//
@RequestBody String body, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId, //
@CookieValue(value = Configuration.AUTH_TOKEN) String token //
) throws SecurityException, QueryException, IOException {
Map<String, Object> node = parseBody(body);
Map<String, Object> data = getData(node, "overlay");
return overlayRestImp.updateOverlay(token, projectId, overlayId, data);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}:downloadSource", method = { RequestMethod.GET },
produces = { MediaType.APPLICATION_JSON_VALUE })
public ResponseEntity<byte[]> getOverlaySource(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId //
) throws SecurityException, QueryException, JsonParseException, JsonMappingException, IOException {
FileEntry file = overlayRestImp.getOverlaySource(token, projectId, overlayId);
MediaType type = MediaType.TEXT_PLAIN;
if (file.getOriginalFileName().endsWith("xml")) {
type = MediaType.APPLICATION_XML;
}
return ResponseEntity
.ok().contentLength(file.getFileContent().length).contentType(type).header("Content-Disposition", "attachment; filename=" + file.getOriginalFileName())
.body(file.getFileContent());
}
/**
* @return the overlayRestImp
* @see #overlayRestImp
*/
public OverlayRestImpl getOverlayRestImp() {
return overlayRestImp;
}
/**
* @param overlayRestImp
* the overlayRestImp to set
* @see #overlayRestImp
*/
public void setOverlayRestImp(OverlayRestImpl overlayRestImp) {
this.overlayRestImp = overlayRestImp;
}
/**
* Default class logger.
*/
@SuppressWarnings("unused")
private Logger logger = Logger.getLogger(OverlayController.class);
@Autowired
private OverlayRestImpl overlayRestImp;
@RequestMapping(value = "/projects/{projectId}/overlays/", method = { RequestMethod.GET }, produces = {
MediaType.APPLICATION_JSON_VALUE })
public List<LayoutView> getOverlayList(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@RequestParam(value = "creator", defaultValue = "") String creator, //
@RequestParam(value = "publicOverlay", defaultValue = "") String publicOverlay //
) throws SecurityException, QueryException {
return overlayRestImp.getOverlayList(token, projectId, creator, publicOverlay);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/", method = { RequestMethod.GET }, produces = {
MediaType.APPLICATION_JSON_VALUE })
public LayoutView getOverlayById(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId//
) throws SecurityException, QueryException {
return overlayRestImp.getOverlayById(token, projectId, overlayId);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/models/{modelId}/bioEntities/", method = {
RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public List<Map<String, Object>> getOverlayElements(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId,
@RequestParam(value = "columns", defaultValue = "") String columns) throws SecurityException, QueryException {
return overlayRestImp.getOverlayElements(token, projectId, Integer.valueOf(overlayId), columns);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/models/{modelId}/bioEntities/reactions/{reactionId}/", method = {
RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public Map<String, Object> getFullReaction(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "modelId") String modelId, //
@PathVariable(value = "overlayId") String overlayId, //
@PathVariable(value = "reactionId") String reactionId, //
@RequestParam(value = "columns", defaultValue = "") String columns //
) throws SecurityException, QueryException {
return overlayRestImp.getOverlayElement(token, projectId, Integer.valueOf(modelId), Integer.valueOf(overlayId),
Integer.valueOf(reactionId), "REACTION", columns);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}/models/{modelId}/bioEntities/elements/{elementId}/", method = {
RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public Map<String, Object> getFullSpecies(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "modelId") String modelId, //
@PathVariable(value = "overlayId") String overlayId, //
@PathVariable(value = "elementId") String reactionId, //
@RequestParam(value = "columns", defaultValue = "") String columns //
) throws SecurityException, QueryException {
return overlayRestImp.getOverlayElement(token, projectId, Integer.valueOf(modelId), Integer.valueOf(overlayId),
Integer.valueOf(reactionId), "ALIAS", columns);
}
@RequestMapping(value = "/projects/{projectId}/overlays/", method = { RequestMethod.POST }, produces = {
MediaType.APPLICATION_JSON_VALUE })
public LayoutView addOverlay(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@RequestParam(value = "name") String name, //
@RequestParam(value = "description") String description, //
@RequestParam(value = "content", defaultValue = "") String content, //
@RequestParam(value = "fileId", defaultValue = "") String fileId, //
@RequestParam(value = "filename") String filename, //
@RequestParam(value = "type", defaultValue = "") String type //
) throws SecurityException, QueryException, IOException {
return overlayRestImp.addOverlay(token, projectId, name, description, content, fileId, filename, type);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}", method = { RequestMethod.DELETE }, produces = {
MediaType.APPLICATION_JSON_VALUE })
public Map<String, Object> removeOverlay(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId //
) throws SecurityException, QueryException, IOException {
return overlayRestImp.removeOverlay(token, projectId, overlayId);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}", method = { RequestMethod.PATCH }, produces = {
MediaType.APPLICATION_JSON_VALUE })
public LayoutView updateOverlay(//
@RequestBody String body, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId, //
@CookieValue(value = Configuration.AUTH_TOKEN) String token //
) throws SecurityException, QueryException, IOException {
Map<String, Object> node = parseBody(body);
Map<String, Object> data = getData(node, "overlay");
return overlayRestImp.updateOverlay(token, projectId, overlayId, data);
}
@RequestMapping(value = "/projects/{projectId}/overlays/{overlayId}:downloadSource", method = {
RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public ResponseEntity<byte[]> getOverlaySource(//
@CookieValue(value = Configuration.AUTH_TOKEN) String token, //
@PathVariable(value = "projectId") String projectId, //
@PathVariable(value = "overlayId") String overlayId //
) throws SecurityException, QueryException, JsonParseException, JsonMappingException, IOException {
FileEntry file = overlayRestImp.getOverlaySource(token, projectId, overlayId);
MediaType type = MediaType.TEXT_PLAIN;
if (file.getOriginalFileName().endsWith("xml")) {
type = MediaType.APPLICATION_XML;
}
return ResponseEntity.ok().contentLength(file.getFileContent().length).contentType(type)
.header("Content-Disposition", "attachment; filename=" + file.getOriginalFileName())
.body(file.getFileContent());
}
/**
* @return the overlayRestImp
* @see #overlayRestImp
*/
public OverlayRestImpl getOverlayRestImp() {
return overlayRestImp;
}
/**
* @param overlayRestImp
* the overlayRestImp to set
* @see #overlayRestImp
*/
public void setOverlayRestImp(OverlayRestImpl overlayRestImp) {
this.overlayRestImp = overlayRestImp;
}
}
\ No newline at end of file
......@@ -17,14 +17,14 @@ import lcsb.mapviewer.api.BaseRestImpl;
import lcsb.mapviewer.api.ObjectNotFoundException;
import lcsb.mapviewer.api.QueryException;
import lcsb.mapviewer.common.Configuration;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.Project;
import lcsb.mapviewer.model.cache.FileEntry;
import lcsb.mapviewer.model.cache.UploadedFileEntry;
import lcsb.mapviewer.model.map.layout.InvalidColorSchemaException;
import lcsb.mapviewer.model.map.layout.Layout;
import lcsb.mapviewer.model.map.model.Model;
import lcsb.mapviewer.model.user.PrivilegeType;
import lcsb.mapviewer.model.user.User;
import lcsb.mapviewer.persist.dao.cache.UploadedFileEntryDao;