Merge branch 'merge-16.0.7' into 'master'

Merge 16.0.7

See merge request !1418

CHANGELOG

@@ -70,6 +70,13 @@ minerva (16.0.2) stable; urgency=medium
 
  -- Piotr Gawron <piotr.gawron@uni.lu>  Mon, 22 Nov 2021 12:00:00 +0200
 
+minerva (16.0.7) stable; urgency=medium
+  * Bug fix: vacuum cron job could cause deadlock and starvation of db
+    connections
+  * Bug fix: mesh parser stopped working after mesh API change
+
+ -- Piotr Gawron <piotr.gawron@uni.lu>  Mon, 17 Jan 2022 12:00:00 +0200
+
 minerva (16.0.6) stable; urgency=medium
   * Bug fix (performance): fetching all bioEntities using plugin API is faster;
     reactions and elements are fetched in parallel (#1605)

annotation/src/main/java/lcsb/mapviewer/annotation/data/MeSH.java

@@ -180,10 +180,14 @@ public class MeSH implements Serializable {
    *          synonyms to add
    */
   public void addSynonyms(final Set<String> synonymsToAdd) {
-    for (final String synonym : synonymsToAdd) {
+    for (String synonym : synonymsToAdd) {
       this.synonyms.add(synonym);
     }
 
   }
 
+  public void removeSynonym(final String synonym) {
+    this.synonyms.remove(synonym);
+  }
+
 }

annotation/src/main/java/lcsb/mapviewer/annotation/services/MeSHParser.java

 package lcsb.mapviewer.annotation.services;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
+import java.net.URLEncoder;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.lang3.SerializationException;
-import org.apache.http.HttpStatus;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Node;
 
-import com.google.gson.Gson;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
 import lcsb.mapviewer.annotation.cache.CachableInterface;
 import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
@@ -28,12 +25,6 @@ import lcsb.mapviewer.common.exception.InvalidArgumentException;
 import lcsb.mapviewer.model.map.MiriamData;
 import lcsb.mapviewer.model.map.MiriamType;
 
-/**
- * Class used for accessing and parsing data from MeSH database.
- * 
- * @author Ayan Rota
- * 
- */
 @Service
 public class MeSHParser extends CachableInterface implements IExternalService {
 
@@ -41,28 +32,27 @@ public class MeSHParser extends CachableInterface implements IExternalService {
    * Prefix used in the DB to identify the cache entry.
    */
   static final String MESH_PREFIX = "mesh:";
-  /**
-   * Url used for searching drugs by name.
-   */
-  private static final String URL_MESH_DATABASE = "https://meshb.nlm.nih.gov/api/record/ui/";
-  /**
-   * Url used for searching mesh terms by synonym.
-   */
-  private static final String URL_SEARCH_BY_SYNONYM = "https://meshb.nlm.nih.gov/api/search/record?searchInField=termDescriptor&sort=&size=20&searchType=exactMatch&searchMethod=FullWord&q=";
-  /**
-   * Default class logger.
-   */
+
+  private static final String SPARQL_QUERY_PREFIX = "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n"
+      + "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n"
+      + "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n"
+      + "PREFIX owl: <http://www.w3.org/2002/07/owl#>\n"
+      + "PREFIX meshv: <http://id.nlm.nih.gov/mesh/vocab#>\n"
+      + "PREFIX mesh: <http://id.nlm.nih.gov/mesh/>\n"
+      + "PREFIX mesh2022: <http://id.nlm.nih.gov/mesh/2022/>\n"
+      + "PREFIX mesh2021: <http://id.nlm.nih.gov/mesh/2021/>\n"
+      + "PREFIX mesh2020: <http://id.nlm.nih.gov/mesh/2020/>\n"
+      + "\n";
+
+  private static final String SPARQL_URL = "https://id.nlm.nih.gov/mesh/sparql?format=JSON&inference=true&offset=0&limit=1000&query=";
+
   private Logger logger = LogManager.getLogger();
 
-  /**
-   * Object that allows to serialize {@link MeSH} elements into xml string and
-   * deserialize xml into {@link MeSH} objects.
-   */
   private XmlSerializer<MeSH> meshSerializer;
 
-  /**
-   * Default constructor.
-   */
+  private ObjectMapper objectMapper = new ObjectMapper();
+
+  @Autowired
   public MeSHParser() {
     super(MeSHParser.class);
     meshSerializer = new XmlSerializer<>(MeSH.class);
@@ -89,7 +79,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
       } else {
         result = super.refreshCacheQuery(query);
       }
-    } catch (final IOException e) {
+    } catch (SourceNotAvailable e) {
       throw new SourceNotAvailable("Problem with accessing Mesh database", e);
     }
     return result;
@@ -112,7 +102,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
    * @throws AnnotatorException
    *           thrown when there is a problem with accessing mesh database
    */
-  public MeSH getMeSH(final MiriamData meshID) throws AnnotatorException {
+  public MeSH getMeSH(final MiriamData meshID) {
     if (meshID == null || meshID.getResource() == null) {
       throw new InvalidArgumentException("mesh cannot be null ");
     }
@@ -135,8 +125,9 @@ public class MeSHParser extends CachableInterface implements IExternalService {
     if (mesh == null) {
       try {
         mesh = getMeSHByIdFromDB(meshID);
-      } catch (final IOException e) {
-        throw new AnnotatorException("Problem with accessing MeSH database", e);
+      } catch (SourceNotAvailable e) {
+        logger.error("Problem with accessing MeSH database", e);
+        return null;
       }
     }
     if (mesh != null) {
@@ -159,102 +150,107 @@ public class MeSHParser extends CachableInterface implements IExternalService {
    * @param meshID
    *          mesh id as Miriam data.
    * @return return as mesh object.
-   * @throws IOException
-   *           thrown when there is problem with accessing web page
-   * @throws AnnotatorException
+   * @throws SourceNotAvailable
    *           thrown when there is a problem with accessing mesh db
    */
-  private MeSH getMeSHByIdFromDB(final MiriamData meshID) throws IOException {
-    try {
-      MeSH result = new MeSH();
-      String page = getWebPageContent(URL_MESH_DATABASE + meshID.getResource());
-
-      Gson gson = new Gson();
-
-      Map<?, ?> gsonObject = new HashMap<String, Object>();
-      gsonObject = (Map<?, ?>) gson.fromJson(page, gsonObject.getClass());
-      Set<String> synonyms = getSynonyms(gsonObject);
-      String name = getName(gsonObject);
-      String description = getDescription(gsonObject);
-      String id = getId(gsonObject);
-      synonyms.remove(name);
-      result.addSynonyms(synonyms);
-      result.setName(name);
-      result.setDescription(description);
-      result.setMeSHId(id);
-
-      return result;
-    } catch (final WrongResponseCodeIOException e) {
-      if (e.getResponseCode() == HttpStatus.SC_NOT_FOUND) {
-        return null;
-      } else {
-        throw e;
-      }
-    }
-  }
+  private MeSH getMeSHByIdFromDB(final MiriamData meshID) throws SourceNotAvailable {
+    MeSH result = new MeSH();
 
-  /**
-   * Extracts name from gson object.
-   *
-   * @param gsonObject
-   *          gson to process
-   * @return name of {@link MeSH} entry
-   */
-  private String getName(final Map<?, ?> gsonObject) {
-    Map<?, ?> descriptorTag = (Map<?, ?>) gsonObject.get("DescriptorName");
-    if (descriptorTag == null) {
-      descriptorTag = (Map<?, ?>) gsonObject.get("SupplementalRecordName");
+    result.setName(getName(meshID));
+    if (result.getName() == null) {
+      return null;
     }
-    return (String) (((Map<?, ?>) descriptorTag.get("String")).get("t"));
+    result.addSynonyms(getSynonyms(meshID));
+    result.setDescription(getDescription(meshID));
+    result.setMeSHId(meshID.getResource());
+    result.removeSynonym(result.getName());
+
+    return result;
   }
 
-  /**
-   * Extracts Mesh id name from gson object.
-   *
-   * @param gsonObject
-   *          gson to process
-   * @return id of {@link MeSH} entry
-   */
-  private String getId(final Map<?, ?> gsonObject) {
-    Map<?, ?> descriptorTag = (Map<?, ?>) gsonObject.get("DescriptorUI");
-    if (descriptorTag == null) {
-      descriptorTag = (Map<?, ?>) gsonObject.get("SupplementalRecordUI");
+  private String getName(final MiriamData meshID) throws SourceNotAvailable {
+    String query = (SPARQL_QUERY_PREFIX
+        + "SELECT * \n"
+        + "FROM <http://id.nlm.nih.gov/mesh>\n"
+        + "WHERE {\n"
+        + "  mesh:MESH_ID rdfs:label ?name .\n"
+        + "}\n").replace("MESH_ID", meshID.getResource());
+
+    try {
+      String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
+      SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
+
+      String name = null;
+      for (Map<String, Map<String, String>> row : result.getResults()) {
+        name = (String) ((Map<?, ?>) row.get("name")).get("value");
+      }
+      return name;
+    } catch (Exception e) {
+      throw new SourceNotAvailable(e);
     }
-    return (String) descriptorTag.get("t");
   }
 
-  /**
-   * Extracts Mesh term description from gson object.
-   *
-   * @param gsonObject
-   *          gson to process
-   * @return description of {@link MeSH} entry
-   */
-  private String getDescription(final Map<?, ?> gsonObject) {
-    Map<?, ?> concepts = (Map<?, ?>) gsonObject.get("_generated");
-    return (String) concepts.get("PreferredConceptScopeNote");
+  private String getDescription(final MiriamData meshID) throws SourceNotAvailable {
+    String query = (SPARQL_QUERY_PREFIX
+        + "SELECT * \n"
+        + "FROM <http://id.nlm.nih.gov/mesh>\n"
+        + "WHERE {\n"
+        + "  mesh:MESH_ID meshv:concept ?concept .\n"
+        + " ?concept meshv:scopeNote ?description .\n"
+        + "}\n").replace("MESH_ID", meshID.getResource());
+
+    try {
+      String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
+      SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
+
+      String description = null;
+      for (Map<String, Map<String, String>> row : result.getResults()) {
+        description = (String) ((Map<?, ?>) row.get("description")).get("value");
+      }
+      return description;
+    } catch (Exception e) {
+      throw new SourceNotAvailable(e);
+    }
   }
 
-  /**
-   * Extracts list of synonyms from gson object.
-   *
-   * @param gsonObject
-   *          gson to process
-   * @return synonyms of {@link MeSH} entry
-   */
-  private Set<String> getSynonyms(final Map<?, ?> gsonObject) {
+  private Set<String> getSynonyms(final MiriamData meshID) throws SourceNotAvailable {
     Set<String> synonyms = new HashSet<>();
-    Map<?, ?> concepts = (Map<?, ?>) gsonObject.get("ConceptList");
-    ArrayList<?> conceptList = (ArrayList<?>) concepts.get("Concept");
-    for (final Object object : conceptList) {
-      Map<?, ?> concept = (Map<?, ?>) object;
-      ArrayList<?> termList = (ArrayList<?>) ((Map<?, ?>) concept.get("TermList")).get("Term");
-      for (final Object object2 : termList) {
-        Map<?, ?> term = (Map<?, ?>) object2;
-        Map<?, ?> synonym = (Map<?, ?>) term.get("String");
-        synonyms.add((String) synonym.get("t"));
+    String query = (SPARQL_QUERY_PREFIX
+        + "SELECT * \n"
+        + "FROM <http://id.nlm.nih.gov/mesh>\n"
+        + "WHERE {\n"
+        + "  mesh:MESH_ID meshv:concept ?concept .\n"
+        + " ?concept meshv:term ?term.\n"
+        + " ?term rdfs:label ?synonym.\n"
+        + "}\n").replace("MESH_ID", meshID.getResource());
+
+    try {
+      String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
+      SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
 
+      for (Map<String, Map<String, String>> row : result.getResults()) {
+        synonyms.add((String) ((Map<?, ?>) row.get("synonym")).get("value"));
       }
+    } catch (Exception e) {
+      throw new SourceNotAvailable(e);
+    }
+    query = (SPARQL_QUERY_PREFIX
+        + "SELECT * \n"
+        + "FROM <http://id.nlm.nih.gov/mesh>\n"
+        + "WHERE {\n"
+        + "  mesh:MESH_ID meshv:concept ?concept .\n"
+        + " ?concept rdfs:label ?conceptName.\n"
+        + "}\n").replace("MESH_ID", meshID.getResource());
+
+    try {
+      String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
+      SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
+
+      for (Map<String, Map<String, String>> row : result.getResults()) {
+        synonyms.add((String) ((Map<?, ?>) row.get("conceptName")).get("value"));
+      }
+    } catch (Exception e) {
+      throw new SourceNotAvailable(e);
     }
     return synonyms;
   }
@@ -289,42 +285,10 @@ public class MeSHParser extends CachableInterface implements IExternalService {
    * @throws AnnotatorException
    *           thrown when there is problem with accessing mesh db
    */
-  public boolean isValidMeshId(final MiriamData meshId) throws AnnotatorException {
+  public boolean isValidMeshId(final MiriamData meshId) {
     if (meshId == null) {
       return false;
     }
     return getMeSH(meshId) != null;
   }
-
-  public List<MeSH> getMeshBySynonym(final String synonym) throws AnnotatorException {
-    try {
-      List<MeSH> result = new ArrayList<>();
-      String page = getWebPageContent(URL_SEARCH_BY_SYNONYM + synonym);
-      Gson gson = new Gson();
-
-      Map<?, ?> gsonObject = new HashMap<String, Object>();
-      gsonObject = (Map<?, ?>) gson.fromJson(page, gsonObject.getClass());
-      Set<MiriamData> synonyms = getIdsBySynonymQuery(gsonObject);
-      for (final MiriamData meshID : synonyms) {
-        result.add(getMeSH(meshID));
-      }
-
-      return result;
-    } catch (final IOException e) {
-      throw new AnnotatorException(e);
-    }
-  }
-
-  private Set<MiriamData> getIdsBySynonymQuery(final Map<?, ?> gsonObject) {
-    Set<MiriamData> result = new HashSet<>();
-    Map<?, ?> hits = (Map<?, ?>) gsonObject.get("hits");
-    ArrayList<?> hitsList = (ArrayList<?>) hits.get("hits");
-    for (final Object object : hitsList) {
-      Map<?, ?> hit = (Map<?, ?>) object;
-      String id = (String) hit.get("_id");
-      result.add(new MiriamData(MiriamType.MESH_2012, id));
-    }
-    return result;
-  }
-
 }

annotation/src/main/java/lcsb/mapviewer/annotation/services/SpqarQLResult.java

+package lcsb.mapviewer.annotation.services;
+
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class SpqarQLResult {
+
+  @JsonProperty
+  private Map<String, List<String>> head;
+
+  private Map<String, List<Map<String, Map<String, String>>>> results;
+
+  public List<Map<String, Map<String, String>>> getResults() {
+    return results.get("bindings");
+  }
+}

annotation/src/test/java/lcsb/mapviewer/annotation/SpringAnnotationTestConfig.java

@@ -4,6 +4,8 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import lcsb.mapviewer.annotation.cache.MockCacheInterface;
 import lcsb.mapviewer.persist.SpringPersistConfig;
 
@@ -16,4 +18,9 @@ public class SpringAnnotationTestConfig {
     return new MockCacheInterface();
   }
 
+  @Bean
+  public ObjectMapper objectMapper() {
+    return new ObjectMapper();
+  }
+
 }

annotation/src/test/java/lcsb/mapviewer/annotation/services/MeSHParserTest.java

@@ -10,9 +10,7 @@ import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
-import java.util.List;
 
-import org.apache.http.HttpStatus;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,7 +20,6 @@ import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
 import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
 import lcsb.mapviewer.annotation.cache.WebPageDownloader;
 import lcsb.mapviewer.annotation.data.MeSH;
-import lcsb.mapviewer.annotation.services.annotators.AnnotatorException;
 import lcsb.mapviewer.common.exception.InvalidArgumentException;
 import lcsb.mapviewer.model.map.MiriamData;
 import lcsb.mapviewer.model.map.MiriamType;
@@ -34,7 +31,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
 
   @Test
   public void testGetMesh() throws Exception {
-    // Parkinson disease
     MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D004298");
     MeSH mesh = meshParser.getMeSH(meshID);
     assertTrue(mesh != null);
@@ -45,15 +41,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
     assertTrue(mesh.getSynonyms().size() > 0);
   }
 
-  @Test
-  public void testGetMeshBySynonym() throws Exception {
-    String synonym = "MPTP";
-    List<MeSH> result = meshParser.getMeshBySynonym(synonym);
-    assertEquals(1, result.size());
-    MeSH mesh = result.get(0);
-    assertTrue(mesh.getSynonyms().contains(synonym));
-  }
-
   @Test
   public void testIsValidMesh() throws Exception {
     MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D004298");
@@ -72,6 +59,12 @@ public class MeSHParserTest extends AnnotationTestFunctions {
     assertFalse(meshParser.isValidMeshId(null));
   }
 
+  @Test
+  public void testIsValidMeshWeirdId() throws Exception {
+    MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "-1");
+    assertFalse(meshParser.isValidMeshId(meshID));
+  }
+
   @Test
   public void testIsValidWithSpace() throws Exception {
     MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "some disease");
@@ -110,19 +103,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
     assertEquals(1, mesh.getSynonyms().size());
   }
 
-  @Test(expected = AnnotatorException.class)
-  public void testGetMEshWithNetworkProblems() throws Exception {
-    // Parkinson disease
-    MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D004298");
-    MeSHParser parserUnderTest = new MeSHParser();
-    WebPageDownloader webPageDownloader = Mockito.mock(WebPageDownloader.class);
-    when(webPageDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
-        .thenThrow(new IOException());
-    parserUnderTest.setWebPageDownloader(webPageDownloader);
-
-    parserUnderTest.getMeSH(meshID);
-  }
-
   @Test
   public void testExternalDBStatus() throws Exception {
     ExternalServiceStatus status = meshParser.getServiceStatus();
@@ -191,20 +171,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
       when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
           .thenThrow(new IOException());
       meshParser.setWebPageDownloader(mockDownloader);
-      assertEquals(ExternalServiceStatusType.DOWN, meshParser.getServiceStatus().getStatus());
-    } finally {
-      meshParser.setWebPageDownloader(downloader);
-    }
-  }
-
-  @Test
-  public void testSimulateChangeStatus() throws Exception {
-    WebPageDownloader downloader = meshParser.getWebPageDownloader();
-    try {
-      WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
-      when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
-          .thenThrow(new WrongResponseCodeIOException(null, HttpStatus.SC_NOT_FOUND));
-      meshParser.setWebPageDownloader(mockDownloader);
       assertEquals(ExternalServiceStatusType.CHANGED, meshParser.getServiceStatus().getStatus());
     } finally {
       meshParser.setWebPageDownloader(downloader);

persist/src/main/java/lcsb/mapviewer/persist/DbUtils.java

@@ -247,7 +247,7 @@ public class DbUtils extends Observable {
         public void execute(final Connection connection) throws SQLException {
           boolean autoCommit = connection.getAutoCommit();
           connection.setAutoCommit(true);
-          connection.prepareStatement("VACUUM FULL").execute();
+          connection.prepareStatement("VACUUM ANALYZE").execute();
           connection.setAutoCommit(autoCommit);
         }
       });

service/src/main/java/lcsb/mapviewer/services/impl/ProjectService.java

@@ -36,7 +36,6 @@ import lcsb.mapviewer.annotation.services.ModelAnnotator;
 import lcsb.mapviewer.annotation.services.ProblematicAnnotation;
 import lcsb.mapviewer.annotation.services.TaxonomyBackend;
 import lcsb.mapviewer.annotation.services.TaxonomySearchException;
-import lcsb.mapviewer.annotation.services.annotators.AnnotatorException;
 import lcsb.mapviewer.annotation.services.annotators.ElementAnnotator;
 import lcsb.mapviewer.commands.ClearColorModelCommand;
 import lcsb.mapviewer.commands.CommandExecutionException;
@@ -1249,16 +1248,11 @@ public class ProjectService implements IProjectService {
     if (params.getOrganism() != null && !params.getOrganism().isEmpty()) {
       organism = new MiriamData(MiriamType.TAXONOMY, params.getOrganism());
     }
-    try {
-      if (meshParser.isValidMeshId(disease)) {
-        project.setDisease(disease);
-      } else {
-        logger.warn("No valid disease is provided for project:" + project.getName());
-      }
-    } catch (final AnnotatorException e1) {
-      logger.warn("Problem with accessing mesh db. More info in logs.", e1);
+    if (meshParser.isValidMeshId(disease)) {
+      project.setDisease(disease);
+    } else {
+      logger.warn("No valid disease is provided for project:" + project.getName());
     }
-
     try {
       if (taxonomyBackend.getNameForTaxonomy(organism) != null) {
         project.setOrganism(organism);