documentation for genome API calls

annotation/src/main/java/lcsb/mapviewer/annotation/services/genome/ReferenceGenomeConnector.java

@@ -78,7 +78,7 @@ public interface ReferenceGenomeConnector {
    * @param version
    *          version of reference genome
    * @param updater
-   *          callback funtion that will be called to update information about
+   *          callback function that will be called to update information about
    *          progress
    * @param async
    *          should the download be performed asynchronously
@@ -100,7 +100,7 @@ public interface ReferenceGenomeConnector {
    * @param referenceGenome
    *          reference genome for which gene mapping is downloaded
    * @param updater
-   *          callback funtion that will be called to update information about
+   *          callback function that will be called to update information about
    *          progress
    * @param async
    *          should the download be performed asynchronously

annotation/src/main/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnector.java

@@ -26,7 +26,7 @@ import lcsb.mapviewer.model.map.MiriamType;
 import lcsb.mapviewer.model.map.layout.*;
 
 /**
- * Implementeation of reference genome connector accessing
+ * Implementation of reference genome connector accessing
  * {@link ReferenceGenomeType#UCSC} database.
  * 
  * @author Piotr Gawron

web/src/main/asciidoc/genomics.adoc

+= Rest API Documentation - Genome
+Piotr Gawron
+v{project-version} {build-time}
+:toc: left
+:sectnums:
+
+== Get all organism available on remote server that have genome data available
+=== CURL sample
+include::{snippets}/genomics/get_organisms/curl-request.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/get_organisms/response-fields.adoc[]
+
+=== Sample Response
+include::{snippets}/genomics/get_organisms/response-body.adoc[]
+
+
+== Get all genome types (remote databases) that support given organism.
+=== CURL sample
+include::{snippets}/genomics/get_genome_types/curl-request.adoc[]
+
+=== Path Parameters
+include::{snippets}/genomics/get_genome_types/path-parameters.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/get_genome_types/response-fields.adoc[]
+
+=== Sample Response
+include::{snippets}/genomics/get_genome_types/response-body.adoc[]
+
+
+== Get all available genome versions for specified organism and genome type
+=== CURL sample
+include::{snippets}/genomics/get_genome_versions/curl-request.adoc[]
+
+=== Path Parameters
+include::{snippets}/genomics/get_genome_versions/path-parameters.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/get_genome_versions/response-fields.adoc[]
+
+=== Sample Response
+include::{snippets}/genomics/get_genome_versions/response-body.adoc[]
+
+
+== Get remote url for specified genome, version, and remote database
+=== CURL sample
+include::{snippets}/genomics/get_genome_remote_url/curl-request.adoc[]
+
+=== Path Parameters
+include::{snippets}/genomics/get_genome_remote_url/path-parameters.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/get_genome_remote_url/response-fields.adoc[]
+
+=== Sample Response
+include::{snippets}/genomics/get_genome_remote_url/response-body.adoc[]
+
+
+== Start downloading genome data from remote database to minerva
+=== CURL sample
+include::{snippets}/genomics/download_genome/curl-request.adoc[]
+
+=== Request Parameters
+include::{snippets}/genomics/download_genome/request-parameters.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/download_genome/response-fields.adoc[]
+
+
+== Get information about specific downloaded genome
+=== CURL sample
+include::{snippets}/genomics/get_genome_information/curl-request.adoc[]
+
+=== Path Parameters
+include::{snippets}/genomics/get_genome_information/path-parameters.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/get_genome_information/response-fields.adoc[]
+
+=== Sample Response
+include::{snippets}/genomics/get_genome_information/response-body.adoc[]
+
+
+== Get information about all downloaded genomes
+=== CURL sample
+include::{snippets}/genomics/get_genomes/curl-request.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/get_genomes/response-fields.adoc[]
+
+=== Sample Response
+include::{snippets}/genomics/get_genomes/response-body.adoc[]
+
+
+== Delete downloaded genome
+=== CURL sample
+include::{snippets}/genomics/delete_genome_information/curl-request.adoc[]
+
+=== Path Parameters
+include::{snippets}/genomics/delete_genome_information/path-parameters.adoc[]
+
+
+== Add gene mapping
+=== Path Parameters
+include::{snippets}/genomics/add_gene_mapping/path-parameters.adoc[]
+
+=== Response Fields
+include::{snippets}/genomics/add_gene_mapping/response-fields.adoc[]
+
+=== CURL sample
+include::{snippets}/genomics/add_gene_mapping/curl-request.adoc[]
+
+
+== Delete gene mapping
+=== CURL sample
+include::{snippets}/genomics/delete_gene_mapping/curl-request.adoc[]
+
+=== Path Parameters
+include::{snippets}/genomics/delete_gene_mapping/path-parameters.adoc[]
+
+

web/src/test/java/lcsb/mapviewer/web/AllIntegrationTests.java

@@ -15,6 +15,7 @@ import org.junit.runners.Suite.SuiteClasses;
     FileControllerIntegrationTest.class,
     FileControllerIntegrationTestWithoutTransaction.class,
     FunctionControllerIntegrationTest.class,
+    GenomicsControllerIntegrationTest.class,
     MapControllerIntegrationTest.class,
     MapControllerIntegrationTestWithoutTransaction.class,
     MiRnaControllerIntegrationTest.class,

web/src/test/java/lcsb/mapviewer/web/GenomicsControllerIntegrationTest.java

+package lcsb.mapviewer.web;
+
+import static org.junit.Assert.assertTrue;
+import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.document;
+import static org.springframework.restdocs.mockmvc.RestDocumentationRequestBuilders.*;
+import static org.springframework.restdocs.payload.PayloadDocumentation.fieldWithPath;
+import static org.springframework.restdocs.payload.PayloadDocumentation.responseFields;
+import static org.springframework.restdocs.request.RequestDocumentation.*;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import java.util.*;
+
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.*;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.mock.web.MockHttpSession;
+import org.springframework.restdocs.payload.FieldDescriptor;
+import org.springframework.test.annotation.Rollback;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.test.web.servlet.RequestBuilder;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.google.gson.JsonParser;
+
+import lcsb.mapviewer.annotation.cache.BigFileCache;
+import lcsb.mapviewer.model.cache.BigFileEntry;
+import lcsb.mapviewer.model.map.MiriamData;
+import lcsb.mapviewer.model.map.MiriamType;
+import lcsb.mapviewer.model.map.layout.*;
+import lcsb.mapviewer.persist.dao.cache.BigFileEntryDao;
+import lcsb.mapviewer.persist.dao.map.layout.ReferenceGenomeDao;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@Transactional
+@Rollback
+@ActiveProfiles("webBigFileAlwaysExistsTestProfile")
+public class GenomicsControllerIntegrationTest extends ControllerIntegrationTest {
+
+  MiriamData ebolaId = new MiriamData(MiriamType.TAXONOMY, "1570291");
+  String ebolaUrl = "ftp://hgdownload.cse.ucsc.edu/goldenPath/eboVir3/bigZips/eboVir3.2bit";
+
+  Logger logger = LogManager.getLogger();
+
+  @Autowired
+  ProjectSnippets snippets;
+
+  @Autowired
+  ReferenceGenomeDao referenceGenomeDao;
+
+  @Autowired
+  BigFileEntryDao bigFileEntryDao;
+
+  @Autowired
+  BigFileCache bigFileCache;
+
+  @Before
+  public void setup() {
+  }
+
+  @After
+  public void tearDown() {
+  }
+
+  @Test
+  public void testGetTaxonomies() throws Exception {
+
+    RequestBuilder request = get("/genomics/taxonomies/");
+
+    String response = mockMvc.perform(request)
+        .andDo(document("genomics/get_organisms",
+            responseFields(fieldWithPath("[]")
+                .description("list of organisms")
+                .type("Array")).andWithPrefix("[].", organismResponseFields())))
+        .andExpect(status().is2xxSuccessful())
+        .andReturn().getResponse().getContentAsString();
+
+    int taxonomies = new JsonParser()
+        .parse(response)
+        .getAsJsonArray().size();
+
+    assertTrue("list of taxonomies shouldn't be empty", taxonomies > 0);
+  }
+
+  private List<FieldDescriptor> organismResponseFields() {
+    return Arrays.asList(
+        fieldWithPath("resource")
+            .description("taxonomy id of the organism")
+            .type("String"),
+        fieldWithPath("annotatorClassName").ignored(),
+        fieldWithPath("id").ignored(),
+        fieldWithPath("link").ignored(),
+        fieldWithPath("type").ignored());
+  }
+
+  @Test
+  public void testGetGenomeTypes() throws Exception {
+
+    RequestBuilder request = get("/genomics/taxonomies/{taxonomyId}/genomeTypes/", "9606");
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/get_genome_types",
+            pathParameters(parameterWithName("taxonomyId").description("organism taxonomy id")),
+            responseFields(
+                fieldWithPath("[]")
+                    .description("list of genome types")
+                    .type("Array"),
+                fieldWithPath("[].type")
+                    .description("type")
+                    .type("String"))))
+        .andExpect(status().is2xxSuccessful());
+  }
+
+  @Test
+  public void testGetAllGenomeVersions() throws Exception {
+
+    RequestBuilder request = get("/genomics/taxonomies/{taxonomyId}/genomeTypes/{genomeType}/versions/", "9606",
+        ReferenceGenomeType.UCSC.name());
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/get_genome_versions",
+            pathParameters(
+                parameterWithName("taxonomyId").description("organism taxonomy id"),
+                parameterWithName("genomeType").description(
+                    "genome type, acceptable values: " + snippets.getOptionsAsString(ReferenceGenomeType.class))),
+            responseFields(
+                fieldWithPath("[]")
+                    .description("list of genome versions")
+                    .type("Array"),
+                fieldWithPath("[].version")
+                    .description("version")
+                    .type("String"))))
+        .andExpect(status().is2xxSuccessful());
+  }
+
+  @Test
+  public void testGetRemoteUrl() throws Exception {
+
+    RequestBuilder request = get(
+        "/genomics/taxonomies/{taxonomyId}/genomeTypes/{genomeType}/versions/{version}:getAvailableRemoteUrls", "9606",
+        ReferenceGenomeType.UCSC.name(), "hg38");
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/get_genome_remote_url",
+            pathParameters(
+                parameterWithName("taxonomyId").description("organism taxonomy id"),
+                parameterWithName("genomeType").description(
+                    "genome type, acceptable values: " + snippets.getOptionsAsString(ReferenceGenomeType.class)),
+                parameterWithName("version").description("genome version")),
+            responseFields(
+                fieldWithPath("[]")
+                    .description("list of urls")
+                    .type("Array"),
+                fieldWithPath("[].url")
+                    .description("url")
+                    .type("String"))))
+        .andExpect(status().is2xxSuccessful());
+  }
+
+  @Test
+  public void testDownloadGenomeDataFromRemote() throws Exception {
+    MockHttpSession session = createSession(BUILT_IN_TEST_ADMIN_LOGIN, BUILT_IN_TEST_ADMIN_PASSWORD);
+
+    String body = EntityUtils.toString(new UrlEncodedFormEntity(Arrays.asList(
+        new BasicNameValuePair("organismId", ebolaId.getResource()),
+        new BasicNameValuePair("type", ReferenceGenomeType.UCSC.name()),
+        new BasicNameValuePair("version", "eboVir3"),
+        new BasicNameValuePair("url", "http://minerva-dev.lcsb.uni.lu/eboVir3.2bit"))));
+
+    RequestBuilder request = post("/genomics/")
+        .contentType(MediaType.APPLICATION_FORM_URLENCODED)
+        .content(body)
+        .session(session);
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/download_genome",
+            requestParameters(
+                parameterWithName("organismId").description("organism taxonomy id"),
+                parameterWithName("type").description(
+                    "genome type, acceptable values: " + snippets.getOptionsAsString(ReferenceGenomeType.class)),
+                parameterWithName("version").description("genome version"),
+                parameterWithName("url").description("url address from where the file should be downloaded")),
+            responseFields()))
+        .andExpect(status().is2xxSuccessful())
+        .andReturn().getResponse().getContentAsString();
+
+  }
+
+  @Test
+  public void testGetDownloadedGenomes() throws Exception {
+    ReferenceGenome genome = new ReferenceGenome();
+    genome.setOrganism(ebolaId);
+    genome.setSourceUrl(ebolaUrl);
+    genome.setDownloadProgress(100.0);
+    genome.setType(ReferenceGenomeType.UCSC);
+    genome.setVersion("eboVir3");
+    referenceGenomeDao.add(genome);
+
+    RequestBuilder request = get("/genomics/");
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/get_genomes",
+            responseFields(fieldWithPath("[]")
+                .description("list of downloaded genomes")
+                .type("Array")).andWithPrefix("[].", genomeInformationResponseFields())))
+        .andExpect(status().is2xxSuccessful())
+        .andReturn().getResponse().getContentAsString();
+
+  }
+
+  @Test
+  public void testGetGenomeInformation() throws Exception {
+    ReferenceGenome genome = new ReferenceGenome();
+    genome.setOrganism(ebolaId);
+    genome.setSourceUrl(ebolaUrl);
+    genome.setDownloadProgress(100.0);
+    genome.setType(ReferenceGenomeType.UCSC);
+    genome.setVersion("eboVir3");
+    referenceGenomeDao.add(genome);
+
+    BigFileEntry entry = new BigFileEntry();
+    entry.setDownloadProgress(100.0);
+    entry.setDownloadDate(Calendar.getInstance());
+    entry.setLocalPath("../");
+    entry.setUrl(ebolaUrl);
+    bigFileEntryDao.add(entry);
+
+    entry.setLocalPath(entry.getLocalPath() + "/" + entry.getId() + "/eboVir3.2bit");
+    bigFileEntryDao.update(entry);
+
+    RequestBuilder request = get(
+        "/genomics/taxonomies/{taxonomyId}/genomeTypes/{genomeType}/versions/{version}/", ebolaId.getResource(),
+        ReferenceGenomeType.UCSC.name(), "eboVir3");
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/get_genome_information",
+            pathParameters(
+                parameterWithName("taxonomyId").description("organism taxonomy id"),
+                parameterWithName("genomeType").description(
+                    "genome type, acceptable values: " + snippets.getOptionsAsString(ReferenceGenomeType.class)),
+                parameterWithName("version").description("genome version")),
+            responseFields(genomeInformationResponseFields())))
+        .andExpect(status().is2xxSuccessful());
+
+  }
+
+  private List<FieldDescriptor> genomeInformationResponseFields() {
+    return Arrays.asList(fieldWithPath("downloadProgress")
+        .description("download progress (0-100%)")
+        .type("Double"),
+        fieldWithPath("geneMapping")
+            .description("list of available gene mappings for given genome")
+            .type("Array"),
+        fieldWithPath("idObject")
+            .description("unique id of genome in minerva")
+            .type("Number"),
+        fieldWithPath("localUrl")
+            .description("url on minerva where local copy of genome can be accessed")
+            .type("String"),
+        fieldWithPath("organism")
+            .description("organism identifier")
+            .type("Object"),
+        fieldWithPath("organism.resource")
+            .description("organism identifier")
+            .type("String"),
+        fieldWithPath("organism.type")
+            .description("organism identifier type (usually " + MiriamType.TAXONOMY + ")")
+            .type("String"),
+        fieldWithPath("organism.annotatorClassName")
+            .ignored(),
+        fieldWithPath("organism.id")
+            .ignored(),
+        fieldWithPath("organism.link")
+            .ignored(),
+        fieldWithPath("sourceUrl")
+            .description("genome source url")
+            .type("String"),
+        fieldWithPath("type")
+            .description("type of genome (database from which it was downloaded)")
+            .type("String"),
+        fieldWithPath("version")
+            .description("genome version")
+            .type("String"));
+  }
+
+  @Test
+  public void deleteGenomeInformation() throws Exception {
+    MockHttpSession session = createSession(BUILT_IN_TEST_ADMIN_LOGIN, BUILT_IN_TEST_ADMIN_PASSWORD);
+
+    ReferenceGenome genome = new ReferenceGenome();
+    genome.setOrganism(ebolaId);
+    genome.setSourceUrl(ebolaUrl);
+    genome.setDownloadProgress(100.0);
+    genome.setType(ReferenceGenomeType.UCSC);
+    genome.setVersion("eboVir3");
+    referenceGenomeDao.add(genome);
+
+    RequestBuilder request = delete("/genomics/{genomeId}/", genome.getId())
+        .session(session);
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/delete_genome_information",
+            pathParameters(
+                parameterWithName("genomeId").description("genome id")),
+            responseFields()))
+        .andExpect(status().is2xxSuccessful());
+
+  }
+
+  @Test
+  public void addGeneMappingInformation() throws Exception {
+    MockHttpSession session = createSession(BUILT_IN_TEST_ADMIN_LOGIN, BUILT_IN_TEST_ADMIN_PASSWORD);
+
+    ReferenceGenome genome = new ReferenceGenome();
+    genome.setOrganism(ebolaId);
+    genome.setSourceUrl(ebolaUrl);
+    genome.setDownloadProgress(100.0);
+    genome.setType(ReferenceGenomeType.UCSC);
+    genome.setVersion("eboVir3");
+    referenceGenomeDao.add(genome);
+
+    String body = EntityUtils.toString(new UrlEncodedFormEntity(Arrays.asList(
+        new BasicNameValuePair("name", "mappingName"),
+        new BasicNameValuePair("url", "https://minerva-dev.lcsb.uni.lu/tmp/refGene.bb"))));
+
+    RequestBuilder request = post("/genomics/{genomeId}/geneMapping/", genome.getId())
+        .contentType(MediaType.APPLICATION_FORM_URLENCODED)
+        .content(body)
+        .session(session);
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/add_gene_mapping",
+            pathParameters(
+                parameterWithName("genomeId").description("genome id")),
+            requestParameters(parameterWithName("name").description("name of the gene-genome mapping"),
+                parameterWithName("url").description("url where file with mapping is located")),
+            responseFields()))
+        .andExpect(status().is2xxSuccessful());
+
+  }
+
+  @Test
+  public void deleteGeneMappingInformation() throws Exception {
+    MockHttpSession session = createSession(BUILT_IN_TEST_ADMIN_LOGIN, BUILT_IN_TEST_ADMIN_PASSWORD);
+
+    ReferenceGenome genome = new ReferenceGenome();
+    genome.setOrganism(ebolaId);
+    genome.setSourceUrl(ebolaUrl);
+    genome.setDownloadProgress(100.0);
+    genome.setType(ReferenceGenomeType.UCSC);
+    genome.setVersion("eboVir3");
+    ReferenceGenomeGeneMapping mapping = new ReferenceGenomeGeneMapping();
+    mapping.setName("x");
+    mapping.setSourceUrl("https://minerva-dev.lcsb.uni.lu/tmp/refGene.bb");
+    genome.addReferenceGenomeGeneMapping(mapping);
+    referenceGenomeDao.add(genome);
+
+    RequestBuilder request = delete("/genomics/{genomeId}/geneMapping/{geneMappingId}/", genome.getId(),
+        genome.getGeneMapping().get(0).getId())
+            .contentType(MediaType.APPLICATION_FORM_URLENCODED)
+            .session(session);
+
+    mockMvc.perform(request)
+        .andDo(document("genomics/delete_gene_mapping",
+            pathParameters(
+                parameterWithName("genomeId").description("genome id"),
+                parameterWithName("geneMappingId").description("gene genome mapping id"))))
+        .andExpect(status().is2xxSuccessful());
+
+  }
+
+}

web/src/test/java/lcsb/mapviewer/web/WebBigFileCacheConfiguration.java

+package lcsb.mapviewer.web;
+
+import static org.mockito.ArgumentMatchers.anyString;
+
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.*;
+
+import lcsb.mapviewer.annotation.cache.BigFileCache;
+import lcsb.mapviewer.model.cache.BigFileEntry;
+import lcsb.mapviewer.persist.DbUtils;
+import lcsb.mapviewer.persist.dao.ConfigurationDao;
+import lcsb.mapviewer.persist.dao.cache.BigFileEntryDao;
+
+@Profile("webBigFileAlwaysExistsTestProfile")
+@Configuration
+public class WebBigFileCacheConfiguration {
+
+  @Autowired
+  private BigFileEntryDao bigFileEntryDao;
+  @Autowired
+  private ConfigurationDao configurationDao;
+  @Autowired
+  private DbUtils dbUtils;
+
+  @Bean
+  @Primary
+  public BigFileCache bigFileCache() throws Exception {
+    BigFileCache cache = new BigFileCache(bigFileEntryDao, configurationDao, dbUtils);
+    BigFileCache mock = Mockito.spy(cache);
+
+    Mockito.doAnswer(new Answer<String>() {
+
+      @Override
+      public String answer(InvocationOnMock invocation) throws Throwable {
+        Object[] args = invocation.getArguments();
+        String remoteUrl = (String) args[0];
+        BigFileEntry entry = bigFileEntryDao.getByUrl(remoteUrl);
+        if (entry != null) {
+          return entry.getLocalPath();
+        } else {
+          return null;
+        }
+      }
+    }).when(mock).getLocalPathForFile(anyString());
+