Skip to content

GitLab

Commit 6a6332d0 authored by Piotr Gawron's avatar Piotr Gawron
Browse files

Merge branch 'potential-deadlock' into 'devel_16.0.x' 

Potential deadlock

See merge request !1416
parents ecc31fd7 b842b29f
Pipeline #51378 passed with stage
in 21 minutes and 34 seconds
Hide whitespace changes
Inline Side-by-side
CHANGELOG
View file @ 6a6332d0
minerva (16.0.7) stable; urgency=medium
* Bug fix (performance): vacuum cron job could cause deadlock and starvation
of db connections
* Bug fix (performance): mesh parser stopped working after mesh API change
-- Piotr Gawron <piotr.gawron@uni.lu> Mon, 17 Jan 2022 12:00:00 +0200
minerva (16.0.6) stable; urgency=medium
* Bug fix (performance): fetching all bioEntities using plugin API is faster;
reactions and elements are fetched in parallel (#1605)
......
CellDesigner-plugin/pom.xml
View file @ 6a6332d0
......@@ -5,7 +5,7 @@
<parent>
<groupId>lcsb.mapviewer</groupId>
<artifactId>parent</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</parent>
<artifactId>CellDesigner-plugin</artifactId>
<!-- dependency from the MapViewer model -->
......@@ -50,19 +50,19 @@
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>model</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</dependency>
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>commons</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</dependency>
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>converter-CellDesigner</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</dependency>
<dependency>
......
annotation/pom.xml
View file @ 6a6332d0
......@@ -5,7 +5,7 @@
<parent>
<groupId>lcsb.mapviewer</groupId>
<artifactId>parent</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</parent>
<artifactId>annotation</artifactId>
<name>Annotation module</name>
......@@ -18,7 +18,7 @@
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>model</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</dependency>
<!-- dependency from the MapViewer dao -->
......@@ -26,7 +26,7 @@
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>persist</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</dependency>
<!-- dependency from the MapViewer cell designer parser (we need access
......@@ -34,13 +34,13 @@
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>converter-CellDesigner</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</dependency>
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>converter</artifactId>
<version>16.0.6</version>
<version>16.0.7</version>
</dependency>
<!-- Log4J2 -->
......
annotation/src/main/java/lcsb/mapviewer/annotation/data/MeSH.java
View file @ 6a6332d0
package lcsb.mapviewer.annotation.data;
import java.io.Serializable;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import javax.xml.bind.annotation.XmlRootElement;
......@@ -67,7 +69,7 @@ public class MeSH implements Serializable {
* @param synonyms
* list of terms used as names for this object.
*/
public MeSH(String meSHId, String name, String description, List<String> synonyms) {
public MeSH(final String meSHId, final String name, final String description, final List<String> synonyms) {
super();
this.name = name;
this.meSHId = meSHId;
......@@ -88,7 +90,7 @@ public class MeSH implements Serializable {
* the name to set
* @see #name
*/
public void setName(String name) {
public void setName(final String name) {
this.name = name;
}
......@@ -105,7 +107,7 @@ public class MeSH implements Serializable {
* the synonyms to set
* @see #synonyms
*/
public void setSynonyms(List<String> synonyms) {
public void setSynonyms(final List<String> synonyms) {
this.synonyms = synonyms;
}
......@@ -120,7 +122,7 @@ public class MeSH implements Serializable {
* @param meSHId
* database identifier
*/
public void setMeSHId(String meSHId) {
public void setMeSHId(final String meSHId) {
this.meSHId = meSHId;
}
......@@ -135,7 +137,7 @@ public class MeSH implements Serializable {
* @param description
* long description.
*/
public void setDescription(String description) {
public void setDescription(final String description) {
this.description = description;
}
......@@ -162,7 +164,7 @@ public class MeSH implements Serializable {
* @param synonym
* synonym to add
*/
public void addSynonym(String synonym) {
public void addSynonym(final String synonym) {
synonyms.add(synonym);
}
......@@ -172,11 +174,15 @@ public class MeSH implements Serializable {
* @param synonymsToAdd
* synonyms to add
*/
public void addSynonyms(Set<String> synonymsToAdd) {
public void addSynonyms(final Set<String> synonymsToAdd) {
for (String synonym : synonymsToAdd) {
this.synonyms.add(synonym);
}
}
public void removeSynonym(final String synonym) {
this.synonyms.remove(synonym);
}
}
annotation/src/main/java/lcsb/mapviewer/annotation/services/MeSHParser.java
View file @ 6a6332d0
package lcsb.mapviewer.annotation.services;
import java.io.IOException;
import java.util.*;
import java.net.URLEncoder;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.SerializationException;
import org.apache.http.HttpStatus;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Node;
import com.google.gson.Gson;
import com.fasterxml.jackson.databind.ObjectMapper;
import lcsb.mapviewer.annotation.cache.*;
import lcsb.mapviewer.annotation.cache.CachableInterface;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.annotation.cache.WebPageDownloader;
import lcsb.mapviewer.annotation.cache.XmlSerializer;
import lcsb.mapviewer.annotation.data.MeSH;
import lcsb.mapviewer.annotation.services.annotators.AnnotatorException;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.map.MiriamData;
import lcsb.mapviewer.model.map.MiriamType;
/**
* Class used for accessing and parsing data from MeSH database.
*
* @author Ayan Rota
*
*/
@Service
public class MeSHParser extends CachableInterface implements IExternalService {
......@@ -32,35 +33,34 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* Prefix used in the DB to identify the cache entry.
*/
static final String MESH_PREFIX = "mesh:";
/**
* Url used for searching drugs by name.
*/
private static final String URL_MESH_DATABASE = "https://meshb.nlm.nih.gov/api/record/ui/";
/**
* Url used for searching mesh terms by synonym.
*/
private static final String URL_SEARCH_BY_SYNONYM = "https://meshb.nlm.nih.gov/api/search/record?searchInField=termDescriptor&sort=&size=20&searchType=exactMatch&searchMethod=FullWord&q=";
/**
* Default class logger.
*/
private static final String SPARQL_QUERY_PREFIX = "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n" +
"PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n" +
"PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n" +
"PREFIX owl: <http://www.w3.org/2002/07/owl#>\n" +
"PREFIX meshv: <http://id.nlm.nih.gov/mesh/vocab#>\n" +
"PREFIX mesh: <http://id.nlm.nih.gov/mesh/>\n" +
"PREFIX mesh2022: <http://id.nlm.nih.gov/mesh/2022/>\n" +
"PREFIX mesh2021: <http://id.nlm.nih.gov/mesh/2021/>\n" +
"PREFIX mesh2020: <http://id.nlm.nih.gov/mesh/2020/>\n" +
"\n";
private static final String SPARQL_URL = "https://id.nlm.nih.gov/mesh/sparql?format=JSON&inference=true&offset=0&limit=1000&query=";
private Logger logger = LogManager.getLogger();
/**
* Object that allows to serialize {@link MeSH} elements into xml string and
* deserialize xml into {@link MeSH} objects.
*/
private XmlSerializer<MeSH> meshSerializer;
/**
* Default constructor.
*/
private ObjectMapper objectMapper = new ObjectMapper();
@Autowired
public MeSHParser() {
super(MeSHParser.class);
meshSerializer = new XmlSerializer<>(MeSH.class);
}
@Override
public Object refreshCacheQuery(Object query) throws SourceNotAvailable {
public Object refreshCacheQuery(final Object query) throws SourceNotAvailable {
Object result = null;
try {
if (query instanceof String) {
......@@ -80,7 +80,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
} else {
result = super.refreshCacheQuery(query);
}
} catch (IOException e) {
} catch (SourceNotAvailable e) {
throw new SourceNotAvailable("Problem with accessing Mesh database", e);
}
return result;
......@@ -92,7 +92,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
}
@Override
protected void setWebPageDownloader(WebPageDownloader webPageDownloader) {
protected void setWebPageDownloader(final WebPageDownloader webPageDownloader) {
super.setWebPageDownloader(webPageDownloader);
}
......@@ -103,7 +103,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* @throws AnnotatorException
* thrown when there is a problem with accessing mesh database
*/
public MeSH getMeSH(MiriamData meshID) throws AnnotatorException {
public MeSH getMeSH(final MiriamData meshID) {
if (meshID == null || meshID.getResource() == null) {
throw new InvalidArgumentException("mesh cannot be null ");
}
......@@ -126,8 +126,9 @@ public class MeSHParser extends CachableInterface implements IExternalService {
if (mesh == null) {
try {
mesh = getMeSHByIdFromDB(meshID);
} catch (IOException e) {
throw new AnnotatorException("Problem with accessing MeSH database", e);
} catch (SourceNotAvailable e) {
logger.error("Problem with accessing MeSH database", e);
return null;
}
}
if (mesh != null) {
......@@ -142,7 +143,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* id as miriam data.
* @return return the object.
*/
public String getIdentifier(MiriamData meshID) {
public String getIdentifier(final MiriamData meshID) {
return MESH_PREFIX + meshID.getResource();
}
......@@ -152,100 +153,108 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* @return return as mesh object.
* @throws IOException
* thrown when there is problem with accessing web page
* @throws SourceNotAvailable
* @throws AnnotatorException
* thrown when there is a problem with accessing mesh db
*/
private MeSH getMeSHByIdFromDB(MiriamData meshID) throws IOException {
try {
MeSH result = new MeSH();
String page = getWebPageContent(URL_MESH_DATABASE + meshID.getResource());
Gson gson = new Gson();
Map<?, ?> gsonObject = new HashMap<String, Object>();
gsonObject = (Map<?, ?>) gson.fromJson(page, gsonObject.getClass());
Set<String> synonyms = getSynonyms(gsonObject);
String name = getName(gsonObject);
String description = getDescription(gsonObject);
String id = getId(gsonObject);
synonyms.remove(name);
result.addSynonyms(synonyms);
result.setName(name);
result.setDescription(description);
result.setMeSHId(id);
return result;
} catch (WrongResponseCodeIOException e) {
if (e.getResponseCode() == HttpStatus.SC_NOT_FOUND) {
return null;
} else {
throw e;
}
}
}
private MeSH getMeSHByIdFromDB(final MiriamData meshID) throws SourceNotAvailable {
MeSH result = new MeSH();
/**
* Extracts name from gson object.
*
* @param gsonObject
* gson to process
* @return name of {@link MeSH} entry
*/
private String getName(Map<?, ?> gsonObject) {
Map<?, ?> descriptorTag = (Map<?, ?>) gsonObject.get("DescriptorName");
if (descriptorTag == null) {
descriptorTag = (Map<?, ?>) gsonObject.get("SupplementalRecordName");
result.setName(getName(meshID));
if (result.getName() == null) {
return null;
}
return (String) (((Map<?, ?>) descriptorTag.get("String")).get("t"));
result.addSynonyms(getSynonyms(meshID));
result.setDescription(getDescription(meshID));
result.setMeSHId(meshID.getResource());
result.removeSynonym(result.getName());
return result;
}
/**
* Extracts Mesh id name from gson object.
*
* @param gsonObject
* gson to process
* @return id of {@link MeSH} entry
*/
private String getId(Map<?, ?> gsonObject) {
Map<?, ?> descriptorTag = (Map<?, ?>) gsonObject.get("DescriptorUI");
if (descriptorTag == null) {
descriptorTag = (Map<?, ?>) gsonObject.get("SupplementalRecordUI");
private String getName(final MiriamData meshID) throws SourceNotAvailable {
String query = (SPARQL_QUERY_PREFIX +
"SELECT * \n" +
"FROM <http://id.nlm.nih.gov/mesh>\n" +
"WHERE {\n" +
" mesh:MESH_ID rdfs:label ?name .\n" +
"}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
String name = null;
for (Map<String, Map<String, String>> row : result.getResults()) {
name = (String) ((Map<?, ?>) row.get("name")).get("value");
}
return name;
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
return (String) descriptorTag.get("t");
}
/**
* Extracts Mesh term description from gson object.
*
* @param gsonObject
* gson to process
* @return description of {@link MeSH} entry
*/
private String getDescription(Map<?, ?> gsonObject) {
Map<?, ?> concepts = (Map<?, ?>) gsonObject.get("_generated");
return (String) concepts.get("PreferredConceptScopeNote");
private String getDescription(final MiriamData meshID) throws SourceNotAvailable {
String query = (SPARQL_QUERY_PREFIX +
"SELECT * \n" +
"FROM <http://id.nlm.nih.gov/mesh>\n" +
"WHERE {\n" +
" mesh:MESH_ID meshv:concept ?concept .\n" +
" ?concept meshv:scopeNote ?description .\n" +
"}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
String description = null;
for (Map<String, Map<String, String>> row : result.getResults()) {
description = (String) ((Map<?, ?>) row.get("description")).get("value");
}
return description;
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
}
/**
* Extracts list of synonyms from gson object.
*
* @param gsonObject
* gson to process
* @return synonyms of {@link MeSH} entry
*/
private Set<String> getSynonyms(Map<?, ?> gsonObject) {
private Set<String> getSynonyms(final MiriamData meshID) throws SourceNotAvailable {
Set<String> synonyms = new HashSet<>();
Map<?, ?> concepts = (Map<?, ?>) gsonObject.get("ConceptList");
ArrayList<?> conceptList = (ArrayList<?>) concepts.get("Concept");
for (Object object : conceptList) {
Map<?, ?> concept = (Map<?, ?>) object;
ArrayList<?> termList = (ArrayList<?>) ((Map<?, ?>) concept.get("TermList")).get("Term");
for (Object object2 : termList) {
Map<?, ?> term = (Map<?, ?>) object2;
Map<?, ?> synonym = (Map<?, ?>) term.get("String");
synonyms.add((String) synonym.get("t"));
String query = (SPARQL_QUERY_PREFIX +
"SELECT * \n" +
"FROM <http://id.nlm.nih.gov/mesh>\n" +
"WHERE {\n" +
" mesh:MESH_ID meshv:concept ?concept .\n" +
" ?concept meshv:term ?term.\n" +
" ?term rdfs:label ?synonym.\n" +
"}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
for (Map<String, Map<String, String>> row : result.getResults()) {
synonyms.add((String) ((Map<?, ?>) row.get("synonym")).get("value"));
}
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
query = (SPARQL_QUERY_PREFIX +
"SELECT * \n" +
"FROM <http://id.nlm.nih.gov/mesh>\n" +
"WHERE {\n" +
" mesh:MESH_ID meshv:concept ?concept .\n" +
" ?concept rdfs:label ?conceptName.\n" +
"}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
for (Map<String, Map<String, String>> row : result.getResults()) {
synonyms.add((String) ((Map<?, ?>) row.get("conceptName")).get("value"));
}
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
return synonyms;
}
......@@ -280,42 +289,10 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* @throws AnnotatorException
* thrown when there is problem with accessing mesh db
*/
public boolean isValidMeshId(MiriamData meshId) throws AnnotatorException {
public boolean isValidMeshId(final MiriamData meshId) {
if (meshId == null) {
return false;
}
return getMeSH(meshId) != null;
}
public List<MeSH> getMeshBySynonym(String synonym) throws AnnotatorException {
try {
List<MeSH> result = new ArrayList<>();
String page = getWebPageContent(URL_SEARCH_BY_SYNONYM + synonym);
Gson gson = new Gson();
Map<?, ?> gsonObject = new HashMap<String, Object>();
gsonObject = (Map<?, ?>) gson.fromJson(page, gsonObject.getClass());
Set<MiriamData> synonyms = getIdsBySynonymQuery(gsonObject);
for (MiriamData meshID : synonyms) {
result.add(getMeSH(meshID));
}
return result;
} catch (IOException e) {
throw new AnnotatorException(e);
}
}
private Set<MiriamData> getIdsBySynonymQuery(Map<?, ?> gsonObject) {
Set<MiriamData> result = new HashSet<>();
Map<?, ?> hits = (Map<?, ?>) gsonObject.get("hits");
ArrayList<?> hitsList = (ArrayList<?>) hits.get("hits");
for (Object object : hitsList) {
Map<?, ?> hit = (Map<?, ?>) object;
String id = (String) hit.get("_id");
result.add(new MiriamData(MiriamType.MESH_2012, id));
}
return result;
}
}
annotation/src/main/java/lcsb/mapviewer/annotation/services/SpqarQLResult.java 0 → 100644
View file @ 6a6332d0
package lcsb.mapviewer.annotation.services;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonProperty;
public class SpqarQLResult {
@JsonProperty
private Map<String, List<String>> head;
private Map<String, List<Map<String, Map<String, String>>>> results;
public List<Map<String, Map<String, String>>> getResults() {
return results.get("bindings");
}
}
annotation/src/test/java/lcsb/mapviewer/annotation/SpringAnnotationTestConfig.java
View file @ 6a6332d0
package lcsb.mapviewer.annotation;
import org.springframework.context.annotation.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import com.fasterxml.jackson.databind.ObjectMapper;
import lcsb.mapviewer.annotation.cache.MockCacheInterface;
import lcsb.mapviewer.persist.SpringPersistConfig;
......@@ -14,4 +18,9 @@ public class SpringAnnotationTestConfig {
return new MockCacheInterface();
}
@Bean
public ObjectMapper objectMapper() {
return new ObjectMapper();
}
}
annotation/src/test/java/lcsb/mapviewer/annotation/services/MeSHParserTest.java
View file @ 6a6332d0
package lcsb.mapviewer.annotation.services;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull