Commit 57ef3e3d authored by Piotr Gawron's avatar Piotr Gawron
Browse files

data mining is incorporated in the model

parent 92e6d5aa
package lcsb.mapviewer.annotation.cache;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
......@@ -78,17 +79,23 @@ public class WebPageDownloader {
// connect to the server, if HTTP_INTERNAL_ERROR occurres then try to
// reconnect HTTP_INTERNAL_ERROR_RECONNECT_TIMES times
while (code == HttpURLConnection.HTTP_INTERNAL_ERROR) {
while (code == HttpURLConnection.HTTP_INTERNAL_ERROR || code == HttpURLConnection.HTTP_NOT_FOUND) {
urlConn = openConnection(accessUrl);
urlConn.setRequestMethod("GET");
urlConn.addRequestProperty("User-Agent", "minerva-framework");
urlConn.connect();
code = urlConn.getResponseCode();
try {
code = urlConn.getResponseCode();
} catch (FileNotFoundException e) {
code = HttpURLConnection.HTTP_NOT_FOUND;
} catch (IOException e) {
code = HttpURLConnection.HTTP_INTERNAL_ERROR;
}
tries++;
if (tries > HTTP_INTERNAL_ERROR_RECONNECT_TIMES) {
break;
}
if (code == HttpURLConnection.HTTP_INTERNAL_ERROR) {
if (code == HttpURLConnection.HTTP_INTERNAL_ERROR || code == HttpURLConnection.HTTP_NOT_FOUND) {
logger.debug("Problem with webpage: " + accessUrl);
logger.debug("Retrying: " + accessUrl);
try {
......@@ -101,7 +108,7 @@ public class WebPageDownloader {
BufferedReader in = new BufferedReader(new InputStreamReader(urlConn.getInputStream()));
result = IOUtils.toString(in);
} catch (IOException e) {
if (e.getClass().equals(IOException.class)) {
if (e.getClass().equals(IOException.class) || e.getClass().equals(FileNotFoundException.class)) {
throw new WrongResponseCodeIOException(e, code);
} else {
throw e;
......
......@@ -258,7 +258,7 @@ public class EnsemblAnnotator extends ElementAnnotator implements IExternalServi
annotatedObject.setSynonyms(sortedSynonyms);
}
} catch (WrongResponseCodeIOException e) {
throw new AnnotatorException(e);
logger.warn(prefix + "Cannot find information for ensembl: " + entrezMiriamData.getResource());
} catch (Exception e) {
throw new AnnotatorException(e);
}
......
......@@ -211,21 +211,24 @@ public class EntrezAnnotator extends ElementAnnotator implements IExternalServic
EntrezData result = entrezSerializer.xmlToObject(super.getCacheNode(ENTREZ_DATA_PREFIX + entrezMiriamData.getResource()));
if (result == null) {
String query = REST_API_URL + entrezMiriamData.getResource();
logger.debug(query);
try {
String content = getWebPageContent(query);
result = new EntrezData();
Node xml = getXmlDocumentFromString(content);
Node fetchResult = getNode("eFetchResult", xml.getChildNodes());
if (fetchResult != null) {
Node errorNode = getNode("Error", fetchResult.getChildNodes());
Node response = getNode("Entrezgene-Set", xml.getChildNodes());
if (response != null) {
Node errorNode = getNode("Error", response.getChildNodes());
if (errorNode != null) {
logger.warn(prefix + " Invalid " + MiriamType.ENTREZ + " identifier: \"" + entrezMiriamData.getResource() + "\"");
return null;
}
} else {
logger.warn(prefix + "Problematic entrez response for identifier: \"" + entrezMiriamData.getResource() + "\"");
return null;
}
Node response = getNode("Entrezgene-Set", xml.getChildNodes());
Node rootNode = getNode("Entrezgene", response.getChildNodes());
Node annotationNode = getNode("Entrezgene_gene", rootNode.getChildNodes());
......
......@@ -15,6 +15,7 @@ import lcsb.mapviewer.annotation.cache.WebPageDownloader;
import lcsb.mapviewer.annotation.services.ExternalServiceStatus;
import lcsb.mapviewer.annotation.services.ExternalServiceStatusType;
import lcsb.mapviewer.annotation.services.IExternalService;
import lcsb.mapviewer.annotation.services.WrongResponseCodeIOException;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.map.AnnotatedObject;
import lcsb.mapviewer.model.map.MiriamData;
......@@ -117,6 +118,8 @@ public class UniprotAnnotator extends ElementAnnotator implements IExternalServi
logger.warn(elementUtils.getElementTag(object) + " Invalid uniprot id: " + uniprotId);
}
}
} catch (WrongResponseCodeIOException exception) {
logger.warn("Cannot find uniprot data for id: " + uniprotId);
} catch (IOException exception) {
throw new AnnotatorException(exception);
}
......
......@@ -72,6 +72,7 @@ public class ChEMBLParserTest extends AnnotationTestFunctions {
assertEquals("CABOZANTINIB", drug.getName());
assertNull(drug.getDescription());
assertEquals(2, drug.getTargets().size());
logger.debug(drug.getSynonyms());
assertTrue(drug.getSynonyms().contains("Cabozantinib"));
} catch (Exception e) {
......
......@@ -284,8 +284,7 @@ public class EnsemblAnnotatorTest extends AnnotationTestFunctions {
Species proteinAlias = new GenericProtein("id");
proteinAlias.addMiriamData(new MiriamData(MiriamType.ENSEMBL, "1234"));
ensemblAnnotator.annotateElement(proteinAlias);
fail("Exception expected");
} catch (AnnotatorException e) {
assertEquals(1, getWarnings().size());
} catch (Exception e) {
e.printStackTrace();
throw e;
......
......@@ -42,6 +42,7 @@ import lcsb.mapviewer.model.map.AnnotatedObject;
import lcsb.mapviewer.model.map.MiriamData;
import lcsb.mapviewer.model.map.SearchIndex;
import lcsb.mapviewer.model.map.compartment.Compartment;
import lcsb.mapviewer.model.map.graph.DataMining;
import lcsb.mapviewer.model.map.layout.graphics.LayerText;
import lcsb.mapviewer.model.map.model.ElementSubmodelConnection;
import lcsb.mapviewer.model.map.model.Model;
......@@ -188,7 +189,14 @@ public abstract class Element implements AnnotatedObject, Serializable {
*/
@Cascade({ CascadeType.ALL })
@OneToMany(fetch = FetchType.EAGER, mappedBy = "source", orphanRemoval = true)
private List<SearchIndex> searchIndexes = new ArrayList<SearchIndex>();
private List<SearchIndex> searchIndexes = new ArrayList<>();
/**
* List of data mining suggestions.
*/
@Cascade({ CascadeType.ALL })
@OneToMany(fetch = FetchType.EAGER, mappedBy = "element", orphanRemoval = true)
private Set<DataMining> dataMining = new HashSet<>();
/**
* Notes describing this element.
......@@ -1057,4 +1065,32 @@ public abstract class Element implements AnnotatedObject, Serializable {
*/
public abstract String getStringType();
/**
* @return the dataMining
* @see #dataMining
*/
public Set<DataMining> getDataMining() {
return dataMining;
}
/**
* @param dataMining
* the dataMining to set
* @see #dataMining
*/
public void setDataMining(Set<DataMining> dataMining) {
this.dataMining = dataMining;
}
/**
* Adds data mining suggestion.
*
* @param dataMining
* data mining to add
*/
public void addDataMining(DataMining dataMining) {
this.dataMining.add(dataMining);
dataMining.setElement(this);
}
}
\ No newline at end of file
package lcsb.mapviewer.persist.dao.map.graph;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.log4j.Logger;
import org.hibernate.Query;
import lcsb.mapviewer.model.map.graph.DataMining;
import lcsb.mapviewer.model.map.species.Element;
import lcsb.mapviewer.persist.dao.BaseDao;
/**
* Data access object for {@link DataMining} class.
*
* @author Piotr Gawron
*
*/
public class DataMiningDao extends BaseDao<DataMining> {
/**
* Default class logger.
*/
@SuppressWarnings("unused")
private static Logger logger = Logger.getLogger(DataMiningDao.class);
/**
* Default constructor.
*/
public DataMiningDao() {
super(DataMining.class);
}
/**
* Returns the data mining elements for unified node.
*
* @param node
* node for which data mining entries should be returned
* @return the data mining elements for unified node
*/
public Collection<DataMining> getMissingConnectionForElement(Element node) {
if (node.getId() == 0) {
return new ArrayList<DataMining>();
} else {
String queryString = "from " + this.getClazz().getSimpleName() + " where element=:node";
Query query = getSession().createQuery(queryString);
@SuppressWarnings("unchecked")
List<DataMining> list = query.setParameter("node", node).list();
return list;
}
}
}
......@@ -36,7 +36,6 @@
<bean id="CacheQueryDao" class="lcsb.mapviewer.persist.dao.cache.CacheQueryDao" parent="BaseDao"/>
<bean id="CacheTypeDao" class="lcsb.mapviewer.persist.dao.cache.CacheTypeDao" parent="BaseDao"/>
<bean id="DataMiningDao" class="lcsb.mapviewer.persist.dao.map.graph.DataMiningDao" parent="BaseDao"/>
<bean id="DataMiningSetDao" class="lcsb.mapviewer.persist.dao.map.graph.DataMiningSetDao" parent="BaseDao"/>
<bean id="AliasDao" class="lcsb.mapviewer.persist.dao.map.layout.alias.AliasDao" parent="BaseDao"/>
......
......@@ -5,8 +5,7 @@ import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({ DataMiningDaoTest.class, //
DataMiningSetDaoTest.class,//
@SuiteClasses({ DataMiningSetDaoTest.class,//
})
public class AllGraphTests {
......
package lcsb.mapviewer.persist.dao.map.graph;
import static org.junit.Assert.assertEquals;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import lcsb.mapviewer.model.Project;
import lcsb.mapviewer.model.map.MiriamData;
import lcsb.mapviewer.model.map.graph.DataMining;
import lcsb.mapviewer.model.map.model.Model;
import lcsb.mapviewer.model.map.model.ModelFullIndexed;
import lcsb.mapviewer.model.map.species.Element;
import lcsb.mapviewer.model.map.species.GenericProtein;
import lcsb.mapviewer.persist.PersistTestFunctions;
public class DataMiningDaoTest extends PersistTestFunctions {
@Autowired
private DataMiningDao dataMiningDao;
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@Test
public void testGetConnections() {
assertEquals(0, dataMiningDao.getMissingConnectionForElement(new GenericProtein("id")).size());
}
@Test
public void testAddRemove() {
try {
String projectId = "Some name";
Project project = new Project();
project.setProjectId(projectId);
Model model = new ModelFullIndexed(null);
Element alias = new GenericProtein("some_el_id");
model.addElement(alias);
project.addModel(model);
projectDao.add(project);
DataMining dm = new DataMining();
dm.setElement(alias);
dm.addReference(new MiriamData());
dm.addSuggestedConnection(new MiriamData());
dataMiningDao.add(dm);
assertEquals(1, dataMiningDao.getMissingConnectionForElement(alias).size());
dataMiningDao.delete(dm);
projectDao.delete(project);
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
@Test
public void testGetConnections2() {
Element species = new GenericProtein("id");
species.setId(1);
assertEquals(0, dataMiningDao.getMissingConnectionForElement(species).size());
}
}
......@@ -24,23 +24,17 @@ import lcsb.mapviewer.common.IProgressUpdater;
import lcsb.mapviewer.common.Pair;
import lcsb.mapviewer.common.XmlParser;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.log.LogType;
import lcsb.mapviewer.model.map.MiriamData;
import lcsb.mapviewer.model.map.MiriamType;
import lcsb.mapviewer.model.map.graph.DataMining;
import lcsb.mapviewer.model.map.graph.DataMiningSet;
import lcsb.mapviewer.model.map.graph.DataMiningType;
import lcsb.mapviewer.model.map.model.Model;
import lcsb.mapviewer.model.map.model.ModelData;
import lcsb.mapviewer.model.map.reaction.Reaction;
import lcsb.mapviewer.model.map.reaction.ReactionNode;
import lcsb.mapviewer.model.map.species.Element;
import lcsb.mapviewer.persist.DbUtils;
import lcsb.mapviewer.persist.dao.map.graph.DataMiningDao;
import lcsb.mapviewer.persist.dao.map.graph.DataMiningSetDao;
import lcsb.mapviewer.services.interfaces.IDataMiningService;
import lcsb.mapviewer.services.interfaces.ILogService;
import lcsb.mapviewer.services.interfaces.ILogService.LogParams;
import lcsb.mapviewer.services.utils.InvalidDataMiningInputFile;
import lcsb.mapviewer.services.view.DataMiningSetView;
......@@ -79,19 +73,6 @@ public class DataMiningService extends XmlParser implements IDataMiningService {
*/
private final Logger logger = Logger.getLogger(DataMiningService.class);
/**
* Data access object for data mining information.
*/
@Autowired
private DataMiningDao missingConnectionDao;
/**
* Utils that help to manage the sessions in custom multithreaded
* implementation.
*/
@Autowired
private DbUtils dbUtils;
/**
* Data access object for {@link DataMiningSet} objects.
*/
......@@ -110,69 +91,6 @@ public class DataMiningService extends XmlParser implements IDataMiningService {
@Autowired
private ILogService logService;
@Override
public Collection<DataMining> getMissingConnections(Element alias) {
return missingConnectionDao.getMissingConnectionForElement(alias);
}
@Override
public void addMissingConnection(DataMining connection) {
missingConnectionDao.add(connection);
Element element = connection.getElement();
String id = element.getName() + " (" + element.getStringType() + ")";
String toString = "";
for (MiriamData md : connection.getSuggestedConnections()) {
toString += md.getResource() + ",";
}
LogParams params = new LogParams().type(LogType.ADD_MISSING_CONNECTION).description("Added connection: " + id + " " + toString).object(connection);
logService.log(params);
}
@Override
public void deleteMissingConnection(DataMining connection, String reason) {
missingConnectionDao.delete(connection);
Element element = connection.getElement();
String id = element.getName() + " (" + element.getStringType() + ")";
String toString = "";
for (MiriamData md : connection.getSuggestedConnections()) {
toString += md.getResource() + ",";
}
LogParams params = new LogParams().type(LogType.REMOVE_MISSING_CONNECTION).description("Connection removed: " + id + " " + toString).object(connection);
logService.log(params);
}
@Override
public void deleteMissingConnection(String connectionId, String reason) {
Integer id = Integer.parseInt(connectionId);
DataMining mc = missingConnectionDao.getById(id);
this.deleteMissingConnection(mc, reason);
}
@Override
public Collection<Element> getSpeciesWithMissingConnectionsByGeneName(Model model, String geneName) {
Set<Element> result = new HashSet<>();
if (geneName == null) {
return result;
}
for (Element element : model.getElements()) {
Collection<DataMining> set = missingConnectionDao.getMissingConnectionForElement(element);
for (DataMining dataMining : set) {
for (MiriamData md : dataMining.getSuggestedConnections()) {
if (md.getResource().equalsIgnoreCase(geneName)) {
result.add(element);
}
}
}
}
return result;
}
@Override
public void removeConnectionsForModel(Model model) {
removeConnectionsForModel(model.getModelData());
}
/**
* @return the logService
*/
......@@ -188,21 +106,6 @@ public class DataMiningService extends XmlParser implements IDataMiningService {
this.logService = logService;
}
/**
* @return the missingConnectionDao
*/
public DataMiningDao getMissingConnectionDao() {
return missingConnectionDao;
}
/**
* @param missingConnectionDao
* the missingConnectionDao to set
*/
public void setMissingConnectionDao(DataMiningDao missingConnectionDao) {
this.missingConnectionDao = missingConnectionDao;
}
/**
* Map that contains cached information if identifiers are hgnc valid names or
* not.
......@@ -233,25 +136,6 @@ public class DataMiningService extends XmlParser implements IDataMiningService {
return status;
}
@Override
public void removeConnectionsForModel(ModelData model) {
Collection<Element> nodes = model.getElements();
dbUtils.setAutoFlush(false);
for (Element node : nodes) {
Collection<DataMining> mcs = missingConnectionDao.getMissingConnectionForElement(node);
for (DataMining missingConnection : mcs) {
missingConnectionDao.delete(missingConnection);
}
}
dbUtils.setAutoFlush(true);
missingConnectionDao.flush();
dbUtils.setAutoFlush(false);
dbUtils.setAutoFlush(true);
missingConnectionDao.flush();
}
@Override
public Set<DataMining> parseData(String filename, DataMiningType type, Collection<Element> nodes, Set<Reaction> reactions, IProgressUpdater updater)
throws IOException, InvalidDataMiningInputFile {
......
......@@ -83,7 +83,6 @@ import lcsb.mapviewer.persist.DbUtils;
import lcsb.mapviewer.persist.dao.ProjectDao;
import lcsb.mapviewer.persist.dao.map.CommentDao;
import lcsb.mapviewer.persist.dao.map.ModelDao;
import lcsb.mapviewer.persist.dao.map.graph.DataMiningDao;
import lcsb.mapviewer.persist.dao.user.UserDao;
import lcsb.mapviewer.services.interfaces.ICommentService;
import lcsb.mapviewer.services.interfaces.IConfigurationService;
......@@ -210,12 +209,6 @@ public class ProjectService implements IProjectService {
@Autowired
private IDataMiningService dataMiningService;
/**
* Data access object for data mining information.
*/
@Autowired
private DataMiningDao missingConnectionDao;
/**
* Data access object for comments.
*/
......@@ -410,7 +403,6 @@ public class ProjectService implements IProjectService {
for (ModelData model : models) {
logger.debug("Remove model: " + model.getId());
commentService.removeCommentsForModel(model);
dataMiningService.removeConnectionsForModel(model);
for (Layout layout : model.getLayouts()) {
try {
......@@ -808,15 +800,12 @@ public class ProjectService implements IProjectService {
double count = 0;
double updateOffset = IProgressUpdater.MAX_PROGRESS * fileCounter / filesCount + CREATION_OF_DATA / ((double) filesCount);
dbUtils.setAutoFlush(false);
for (DataMining missingConnection : result) {
missingConnection.setType(dmSet.getType());
missingConnectionDao.add(missingConnection);
missingConnection.getElement().addDataMining(missingConnection);
count++;
progressUpdater.setProgress(updateOffset + IProgressUpdater.MAX_PROGRESS * count / size * UPLOAD_OF_DATA);
}
dbUtils.setAutoFlush(true);
missingConnectionDao.flush();
}
}
......@@ -1044,23 +1033,6 @@ public class ProjectService implements IProjectService {
}
/**
* @return the missingConnectionDao
* @see #missingConnectionDao
*/
public DataMiningDao getMissingConnectionDao() {
return missingConnectionDao;
}
/**
* @param missingConnectionDao
* the missingConnectionDao to set
* @see #missingConnectionDao
*/
public void setMissingConnectionDao(DataMiningDao missingConnectionDao) {
this.missingConnectionDao = missingConnectionDao;
}
/**
* @return the modelDao
* @see #modelDao
......@@ -1230,7 +1202,7 @@ public class ProjectService implements IProjectService {
for (ProblematicAnnotation improperAnnotation : missingAnnotations) {
res.add(improperAnnotation.toString());
}
for (String message: res) {
for (String message : res) {
logger.warn(message);
}
logger.debug("Analyze finished");
......
......@@ -8,8 +8,6 @@ import lcsb.mapviewer.common.IProgressUpdater;
import lcsb.mapviewer.model.map.graph.DataMining;
import lcsb.mapviewer.model.map.graph.DataMiningSet;
import lcsb.mapviewer.model.map.graph.DataMiningType;
import lcsb.mapviewer.model.map.model.Model;
import lcsb.mapviewer.model.map.model.ModelData;