Commit a53a9c1c authored by Piotr Gawron's avatar Piotr Gawron
Browse files

Merge branch 'merge-16.0.7' into 'master'

Merge 16.0.7

See merge request !1418
parents 51663667 be3cf784
Pipeline #51498 passed with stage
in 19 minutes and 35 seconds
......@@ -70,6 +70,13 @@ minerva (16.0.2) stable; urgency=medium
-- Piotr Gawron <piotr.gawron@uni.lu> Mon, 22 Nov 2021 12:00:00 +0200
minerva (16.0.7) stable; urgency=medium
* Bug fix: vacuum cron job could cause deadlock and starvation of db
connections
* Bug fix: mesh parser stopped working after mesh API change
-- Piotr Gawron <piotr.gawron@uni.lu> Mon, 17 Jan 2022 12:00:00 +0200
minerva (16.0.6) stable; urgency=medium
* Bug fix (performance): fetching all bioEntities using plugin API is faster;
reactions and elements are fetched in parallel (#1605)
......
......@@ -180,10 +180,14 @@ public class MeSH implements Serializable {
* synonyms to add
*/
public void addSynonyms(final Set<String> synonymsToAdd) {
for (final String synonym : synonymsToAdd) {
for (String synonym : synonymsToAdd) {
this.synonyms.add(synonym);
}
}
public void removeSynonym(final String synonym) {
this.synonyms.remove(synonym);
}
}
package lcsb.mapviewer.annotation.services;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.net.URLEncoder;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.SerializationException;
import org.apache.http.HttpStatus;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Node;
import com.google.gson.Gson;
import com.fasterxml.jackson.databind.ObjectMapper;
import lcsb.mapviewer.annotation.cache.CachableInterface;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
......@@ -28,12 +25,6 @@ import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.map.MiriamData;
import lcsb.mapviewer.model.map.MiriamType;
/**
* Class used for accessing and parsing data from MeSH database.
*
* @author Ayan Rota
*
*/
@Service
public class MeSHParser extends CachableInterface implements IExternalService {
......@@ -41,28 +32,27 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* Prefix used in the DB to identify the cache entry.
*/
static final String MESH_PREFIX = "mesh:";
/**
* Url used for searching drugs by name.
*/
private static final String URL_MESH_DATABASE = "https://meshb.nlm.nih.gov/api/record/ui/";
/**
* Url used for searching mesh terms by synonym.
*/
private static final String URL_SEARCH_BY_SYNONYM = "https://meshb.nlm.nih.gov/api/search/record?searchInField=termDescriptor&sort=&size=20&searchType=exactMatch&searchMethod=FullWord&q=";
/**
* Default class logger.
*/
private static final String SPARQL_QUERY_PREFIX = "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n"
+ "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n"
+ "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n"
+ "PREFIX owl: <http://www.w3.org/2002/07/owl#>\n"
+ "PREFIX meshv: <http://id.nlm.nih.gov/mesh/vocab#>\n"
+ "PREFIX mesh: <http://id.nlm.nih.gov/mesh/>\n"
+ "PREFIX mesh2022: <http://id.nlm.nih.gov/mesh/2022/>\n"
+ "PREFIX mesh2021: <http://id.nlm.nih.gov/mesh/2021/>\n"
+ "PREFIX mesh2020: <http://id.nlm.nih.gov/mesh/2020/>\n"
+ "\n";
private static final String SPARQL_URL = "https://id.nlm.nih.gov/mesh/sparql?format=JSON&inference=true&offset=0&limit=1000&query=";
private Logger logger = LogManager.getLogger();
/**
* Object that allows to serialize {@link MeSH} elements into xml string and
* deserialize xml into {@link MeSH} objects.
*/
private XmlSerializer<MeSH> meshSerializer;
/**
* Default constructor.
*/
private ObjectMapper objectMapper = new ObjectMapper();
@Autowired
public MeSHParser() {
super(MeSHParser.class);
meshSerializer = new XmlSerializer<>(MeSH.class);
......@@ -89,7 +79,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
} else {
result = super.refreshCacheQuery(query);
}
} catch (final IOException e) {
} catch (SourceNotAvailable e) {
throw new SourceNotAvailable("Problem with accessing Mesh database", e);
}
return result;
......@@ -112,7 +102,7 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* @throws AnnotatorException
* thrown when there is a problem with accessing mesh database
*/
public MeSH getMeSH(final MiriamData meshID) throws AnnotatorException {
public MeSH getMeSH(final MiriamData meshID) {
if (meshID == null || meshID.getResource() == null) {
throw new InvalidArgumentException("mesh cannot be null ");
}
......@@ -135,8 +125,9 @@ public class MeSHParser extends CachableInterface implements IExternalService {
if (mesh == null) {
try {
mesh = getMeSHByIdFromDB(meshID);
} catch (final IOException e) {
throw new AnnotatorException("Problem with accessing MeSH database", e);
} catch (SourceNotAvailable e) {
logger.error("Problem with accessing MeSH database", e);
return null;
}
}
if (mesh != null) {
......@@ -159,102 +150,107 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* @param meshID
* mesh id as Miriam data.
* @return return as mesh object.
* @throws IOException
* thrown when there is problem with accessing web page
* @throws AnnotatorException
* @throws SourceNotAvailable
* thrown when there is a problem with accessing mesh db
*/
private MeSH getMeSHByIdFromDB(final MiriamData meshID) throws IOException {
try {
MeSH result = new MeSH();
String page = getWebPageContent(URL_MESH_DATABASE + meshID.getResource());
Gson gson = new Gson();
Map<?, ?> gsonObject = new HashMap<String, Object>();
gsonObject = (Map<?, ?>) gson.fromJson(page, gsonObject.getClass());
Set<String> synonyms = getSynonyms(gsonObject);
String name = getName(gsonObject);
String description = getDescription(gsonObject);
String id = getId(gsonObject);
synonyms.remove(name);
result.addSynonyms(synonyms);
result.setName(name);
result.setDescription(description);
result.setMeSHId(id);
return result;
} catch (final WrongResponseCodeIOException e) {
if (e.getResponseCode() == HttpStatus.SC_NOT_FOUND) {
return null;
} else {
throw e;
}
}
}
private MeSH getMeSHByIdFromDB(final MiriamData meshID) throws SourceNotAvailable {
MeSH result = new MeSH();
/**
* Extracts name from gson object.
*
* @param gsonObject
* gson to process
* @return name of {@link MeSH} entry
*/
private String getName(final Map<?, ?> gsonObject) {
Map<?, ?> descriptorTag = (Map<?, ?>) gsonObject.get("DescriptorName");
if (descriptorTag == null) {
descriptorTag = (Map<?, ?>) gsonObject.get("SupplementalRecordName");
result.setName(getName(meshID));
if (result.getName() == null) {
return null;
}
return (String) (((Map<?, ?>) descriptorTag.get("String")).get("t"));
result.addSynonyms(getSynonyms(meshID));
result.setDescription(getDescription(meshID));
result.setMeSHId(meshID.getResource());
result.removeSynonym(result.getName());
return result;
}
/**
* Extracts Mesh id name from gson object.
*
* @param gsonObject
* gson to process
* @return id of {@link MeSH} entry
*/
private String getId(final Map<?, ?> gsonObject) {
Map<?, ?> descriptorTag = (Map<?, ?>) gsonObject.get("DescriptorUI");
if (descriptorTag == null) {
descriptorTag = (Map<?, ?>) gsonObject.get("SupplementalRecordUI");
private String getName(final MiriamData meshID) throws SourceNotAvailable {
String query = (SPARQL_QUERY_PREFIX
+ "SELECT * \n"
+ "FROM <http://id.nlm.nih.gov/mesh>\n"
+ "WHERE {\n"
+ " mesh:MESH_ID rdfs:label ?name .\n"
+ "}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
String name = null;
for (Map<String, Map<String, String>> row : result.getResults()) {
name = (String) ((Map<?, ?>) row.get("name")).get("value");
}
return name;
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
return (String) descriptorTag.get("t");
}
/**
* Extracts Mesh term description from gson object.
*
* @param gsonObject
* gson to process
* @return description of {@link MeSH} entry
*/
private String getDescription(final Map<?, ?> gsonObject) {
Map<?, ?> concepts = (Map<?, ?>) gsonObject.get("_generated");
return (String) concepts.get("PreferredConceptScopeNote");
private String getDescription(final MiriamData meshID) throws SourceNotAvailable {
String query = (SPARQL_QUERY_PREFIX
+ "SELECT * \n"
+ "FROM <http://id.nlm.nih.gov/mesh>\n"
+ "WHERE {\n"
+ " mesh:MESH_ID meshv:concept ?concept .\n"
+ " ?concept meshv:scopeNote ?description .\n"
+ "}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
String description = null;
for (Map<String, Map<String, String>> row : result.getResults()) {
description = (String) ((Map<?, ?>) row.get("description")).get("value");
}
return description;
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
}
/**
* Extracts list of synonyms from gson object.
*
* @param gsonObject
* gson to process
* @return synonyms of {@link MeSH} entry
*/
private Set<String> getSynonyms(final Map<?, ?> gsonObject) {
private Set<String> getSynonyms(final MiriamData meshID) throws SourceNotAvailable {
Set<String> synonyms = new HashSet<>();
Map<?, ?> concepts = (Map<?, ?>) gsonObject.get("ConceptList");
ArrayList<?> conceptList = (ArrayList<?>) concepts.get("Concept");
for (final Object object : conceptList) {
Map<?, ?> concept = (Map<?, ?>) object;
ArrayList<?> termList = (ArrayList<?>) ((Map<?, ?>) concept.get("TermList")).get("Term");
for (final Object object2 : termList) {
Map<?, ?> term = (Map<?, ?>) object2;
Map<?, ?> synonym = (Map<?, ?>) term.get("String");
synonyms.add((String) synonym.get("t"));
String query = (SPARQL_QUERY_PREFIX
+ "SELECT * \n"
+ "FROM <http://id.nlm.nih.gov/mesh>\n"
+ "WHERE {\n"
+ " mesh:MESH_ID meshv:concept ?concept .\n"
+ " ?concept meshv:term ?term.\n"
+ " ?term rdfs:label ?synonym.\n"
+ "}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
for (Map<String, Map<String, String>> row : result.getResults()) {
synonyms.add((String) ((Map<?, ?>) row.get("synonym")).get("value"));
}
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
query = (SPARQL_QUERY_PREFIX
+ "SELECT * \n"
+ "FROM <http://id.nlm.nih.gov/mesh>\n"
+ "WHERE {\n"
+ " mesh:MESH_ID meshv:concept ?concept .\n"
+ " ?concept rdfs:label ?conceptName.\n"
+ "}\n").replace("MESH_ID", meshID.getResource());
try {
String page = getWebPageContent(SPARQL_URL + URLEncoder.encode(query, "UTF-8"));
SpqarQLResult result = objectMapper.readValue(page, SpqarQLResult.class);
for (Map<String, Map<String, String>> row : result.getResults()) {
synonyms.add((String) ((Map<?, ?>) row.get("conceptName")).get("value"));
}
} catch (Exception e) {
throw new SourceNotAvailable(e);
}
return synonyms;
}
......@@ -289,42 +285,10 @@ public class MeSHParser extends CachableInterface implements IExternalService {
* @throws AnnotatorException
* thrown when there is problem with accessing mesh db
*/
public boolean isValidMeshId(final MiriamData meshId) throws AnnotatorException {
public boolean isValidMeshId(final MiriamData meshId) {
if (meshId == null) {
return false;
}
return getMeSH(meshId) != null;
}
public List<MeSH> getMeshBySynonym(final String synonym) throws AnnotatorException {
try {
List<MeSH> result = new ArrayList<>();
String page = getWebPageContent(URL_SEARCH_BY_SYNONYM + synonym);
Gson gson = new Gson();
Map<?, ?> gsonObject = new HashMap<String, Object>();
gsonObject = (Map<?, ?>) gson.fromJson(page, gsonObject.getClass());
Set<MiriamData> synonyms = getIdsBySynonymQuery(gsonObject);
for (final MiriamData meshID : synonyms) {
result.add(getMeSH(meshID));
}
return result;
} catch (final IOException e) {
throw new AnnotatorException(e);
}
}
private Set<MiriamData> getIdsBySynonymQuery(final Map<?, ?> gsonObject) {
Set<MiriamData> result = new HashSet<>();
Map<?, ?> hits = (Map<?, ?>) gsonObject.get("hits");
ArrayList<?> hitsList = (ArrayList<?>) hits.get("hits");
for (final Object object : hitsList) {
Map<?, ?> hit = (Map<?, ?>) object;
String id = (String) hit.get("_id");
result.add(new MiriamData(MiriamType.MESH_2012, id));
}
return result;
}
}
package lcsb.mapviewer.annotation.services;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonProperty;
public class SpqarQLResult {
@JsonProperty
private Map<String, List<String>> head;
private Map<String, List<Map<String, Map<String, String>>>> results;
public List<Map<String, Map<String, String>>> getResults() {
return results.get("bindings");
}
}
......@@ -4,6 +4,8 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import com.fasterxml.jackson.databind.ObjectMapper;
import lcsb.mapviewer.annotation.cache.MockCacheInterface;
import lcsb.mapviewer.persist.SpringPersistConfig;
......@@ -16,4 +18,9 @@ public class SpringAnnotationTestConfig {
return new MockCacheInterface();
}
@Bean
public ObjectMapper objectMapper() {
return new ObjectMapper();
}
}
......@@ -10,9 +10,7 @@ import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.List;
import org.apache.http.HttpStatus;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -22,7 +20,6 @@ import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.annotation.cache.WebPageDownloader;
import lcsb.mapviewer.annotation.data.MeSH;
import lcsb.mapviewer.annotation.services.annotators.AnnotatorException;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.map.MiriamData;
import lcsb.mapviewer.model.map.MiriamType;
......@@ -34,7 +31,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
@Test
public void testGetMesh() throws Exception {
// Parkinson disease
MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D004298");
MeSH mesh = meshParser.getMeSH(meshID);
assertTrue(mesh != null);
......@@ -45,15 +41,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
assertTrue(mesh.getSynonyms().size() > 0);
}
@Test
public void testGetMeshBySynonym() throws Exception {
String synonym = "MPTP";
List<MeSH> result = meshParser.getMeshBySynonym(synonym);
assertEquals(1, result.size());
MeSH mesh = result.get(0);
assertTrue(mesh.getSynonyms().contains(synonym));
}
@Test
public void testIsValidMesh() throws Exception {
MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D004298");
......@@ -72,6 +59,12 @@ public class MeSHParserTest extends AnnotationTestFunctions {
assertFalse(meshParser.isValidMeshId(null));
}
@Test
public void testIsValidMeshWeirdId() throws Exception {
MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "-1");
assertFalse(meshParser.isValidMeshId(meshID));
}
@Test
public void testIsValidWithSpace() throws Exception {
MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "some disease");
......@@ -110,19 +103,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
assertEquals(1, mesh.getSynonyms().size());
}
@Test(expected = AnnotatorException.class)
public void testGetMEshWithNetworkProblems() throws Exception {
// Parkinson disease
MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D004298");
MeSHParser parserUnderTest = new MeSHParser();
WebPageDownloader webPageDownloader = Mockito.mock(WebPageDownloader.class);
when(webPageDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
.thenThrow(new IOException());
parserUnderTest.setWebPageDownloader(webPageDownloader);
parserUnderTest.getMeSH(meshID);
}
@Test
public void testExternalDBStatus() throws Exception {
ExternalServiceStatus status = meshParser.getServiceStatus();
......@@ -191,20 +171,6 @@ public class MeSHParserTest extends AnnotationTestFunctions {
when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
.thenThrow(new IOException());
meshParser.setWebPageDownloader(mockDownloader);
assertEquals(ExternalServiceStatusType.DOWN, meshParser.getServiceStatus().getStatus());
} finally {
meshParser.setWebPageDownloader(downloader);
}
}
@Test
public void testSimulateChangeStatus() throws Exception {
WebPageDownloader downloader = meshParser.getWebPageDownloader();
try {
WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
.thenThrow(new WrongResponseCodeIOException(null, HttpStatus.SC_NOT_FOUND));
meshParser.setWebPageDownloader(mockDownloader);
assertEquals(ExternalServiceStatusType.CHANGED, meshParser.getServiceStatus().getStatus());
} finally {
meshParser.setWebPageDownloader(downloader);
......
......@@ -247,7 +247,7 @@ public class DbUtils extends Observable {
public void execute(final Connection connection) throws SQLException {
boolean autoCommit = connection.getAutoCommit();
connection.setAutoCommit(true);
connection.prepareStatement("VACUUM FULL").execute();
connection.prepareStatement("VACUUM ANALYZE").execute();
connection.setAutoCommit(autoCommit);
}
});
......
......@@ -36,7 +36,6 @@ import lcsb.mapviewer.annotation.services.ModelAnnotator;
import lcsb.mapviewer.annotation.services.ProblematicAnnotation;
import lcsb.mapviewer.annotation.services.TaxonomyBackend;
import lcsb.mapviewer.annotation.services.TaxonomySearchException;
import lcsb.mapviewer.annotation.services.annotators.AnnotatorException;
import lcsb.mapviewer.annotation.services.annotators.ElementAnnotator;
import lcsb.mapviewer.commands.ClearColorModelCommand;
import lcsb.mapviewer.commands.CommandExecutionException;
......@@ -1249,16 +1248,11 @@ public class ProjectService implements IProjectService {
if (params.getOrganism() != null && !params.getOrganism().isEmpty()) {
organism = new MiriamData(MiriamType.TAXONOMY, params.getOrganism());
}
try {
if (meshParser.isValidMeshId(disease)) {
project.setDisease(disease);
} else {
logger.warn("No valid disease is provided for project:" + project.getName());
}
} catch (final AnnotatorException e1) {
logger.warn("Problem with accessing mesh db. More info in logs.", e1);
if (meshParser.isValidMeshId(disease)) {
project.setDisease(disease);
} else {
logger.warn("No valid disease is provided for project:" + project.getName());
}
try {
if (taxonomyBackend.getNameForTaxonomy(organism) != null) {
project.setOrganism(organism);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment