Commit 0642fbac authored by Piotr Gawron's avatar Piotr Gawron
Browse files

Merge branch 'multithread-test-issue' into 'master'

wait function implemented for db cache tests

See merge request !453
parents e03ef028 c4a9b6db
Pipeline #6799 passed with stage
in 7 minutes and 38 seconds
......@@ -2,6 +2,8 @@ package lcsb.mapviewer.annotation.cache;
import java.lang.reflect.Constructor;
import java.util.Calendar;
import java.util.Collection;
import java.util.LinkedList;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
......@@ -143,8 +145,8 @@ public class PermanentDatabaseLevelCache extends XmlParser
if (object instanceof CachableInterface) {
cachableInterface = (CachableInterface) object;
} else {
logger.fatal(
"Invalid class type: " + object.getClass() + ". Class cannot be cast into " + CachableInterface.class);
logger.fatal("Invalid class type: " + object.getClass() + " [" + object.toString()
+ "]. Class cannot be cast into " + CachableInterface.class);
}
applicationContext.getAutowireCapableBeanFactory().autowireBean(object);
......@@ -563,16 +565,6 @@ public class PermanentDatabaseLevelCache extends XmlParser
return ((ScheduledThreadPoolExecutor) cacheRefreshService).getQueue().size();
}
@Override
public boolean refreshIsBusy() {
return getRefreshPendingQueueSize() != 0 || getRefreshExecutingTasksSize() != 0;
}
@Override
public int getRefreshExecutingTasksSize() {
return ((ScheduledThreadPoolExecutor) cacheRefreshService).getActiveCount();
}
@Override
public CacheQueryDao getCacheQueryDao() {
return cacheQueryDao;
......@@ -584,9 +576,24 @@ public class PermanentDatabaseLevelCache extends XmlParser
}
@Override
public boolean databaseCacheIsBusy() {
return refreshIsBusy() || ((ScheduledThreadPoolExecutor) service).getActiveCount() > 0
|| ((ScheduledThreadPoolExecutor) service).getQueue().size() > 0;
public void waitToFinishTasks() throws InterruptedException, ExecutionException {
Collection<Future<?>> futures = new LinkedList<Future<?>>();
futures.add(cacheRefreshService.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
return null;
}
}));
futures.add(service.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
return null;
}
}));
for (Future<?> future:futures) {
future.get();
}
}
}
\ No newline at end of file
package lcsb.mapviewer.annotation.cache;
import java.util.concurrent.ExecutionException;
import lcsb.mapviewer.persist.dao.cache.CacheQueryDao;
/**
......@@ -20,25 +22,11 @@ public interface PermanentDatabaseLevelCacheInterface extends QueryCacheInterfac
int getRefreshPendingQueueSize();
/**
* Returns true if refreshing of entries in database is in progress.
*
* @return true if refreshing of entries in database is in progress.
*/
boolean refreshIsBusy();
/**
* Returns approximate number of refresh tasks that are currently executed.
*
* @return approximate number of refresh tasks that are currently executed.
*/
int getRefreshExecutingTasksSize();
/**
* Returns true if database cache has any task in progress.
*
* @return true if database cache has any task in progress.
* Waits for all tasks in the cache to finish (refresh/get/etc).
* @throws ExecutionException
* @throws InterruptedException
*/
boolean databaseCacheIsBusy();
void waitToFinishTasks() throws InterruptedException, ExecutionException;
/**
* Returns {@link CacheQueryDao} used by the cache.
......
......@@ -283,12 +283,4 @@ public abstract class AnnotationTestFunctions extends AbstractTransactionalJUnit
throw exception;
}
protected void waitForRefreshCacheQueueToEmpty() throws InterruptedException {
while (cache.refreshIsBusy()) {
logger.debug("Waiting for refresh queue to empty. " + cache.getRefreshPendingQueueSize() + " pending. "
+ cache.getRefreshExecutingTasksSize() + " tasks are executed.");
Thread.sleep(300);
}
}
}
......@@ -18,6 +18,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.annotation.Rollback;
import org.w3c.dom.Node;
......@@ -34,6 +35,9 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
Logger logger = Logger.getLogger(PermanentDatabaseLevelCacheTest.class);
@Autowired
private PermanentDatabaseLevelCacheInterface permanentDatabaseLevelCache;
@Before
public void setUp() throws Exception {
}
......@@ -115,7 +119,6 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
@Test(timeout = 15000)
public void testRefresh() throws Exception {
try {
waitForRefreshCacheQueueToEmpty();
CacheType invalidType = cacheTypeDao.getByClassName(MockCacheInterface.class.getCanonicalName());
String query = "test query";
......@@ -129,7 +132,7 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
// wasn't updated)
assertEquals(val, res);
// sleep a bit to wait for the second thread
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
// for the second time it should change
dbUtils.createSessionForCurrentThread();
res = cache.getStringByQuery(query, invalidType);
......@@ -300,9 +303,8 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
cache.invalidateByQuery(query, invalidType);
while (cache.databaseCacheIsBusy()) {
Thread.sleep(10);
}
cache.waitToFinishTasks();
assertEquals(1, getFatals().size());
} catch (Exception e) {
......@@ -337,9 +339,8 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
cache.invalidateByQuery(query, invalidType);
while (cache.databaseCacheIsBusy()) {
Thread.sleep(10);
}
cache.waitToFinishTasks();
assertEquals(1, getFatals().size());
} catch (Exception e) {
......@@ -373,9 +374,7 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
cache.invalidateByQuery(query, type);
while (cache.refreshIsBusy()) {
Thread.sleep(10);
}
permanentDatabaseLevelCache.waitToFinishTasks();
assertEquals(1, getErrors().size());
} catch (Exception e) {
......@@ -409,9 +408,7 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
cache.invalidateByQuery(query, type);
while (cache.getRefreshExecutingTasksSize() + cache.getRefreshPendingQueueSize() > 0) {
Thread.sleep(10);
}
permanentDatabaseLevelCache.waitToFinishTasks();
assertEquals(0, getErrors().size());
assertEquals(0, getFatals().size());
......@@ -427,8 +424,6 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
public void testRefreshQueryWithExceptionThrown() throws Exception {
CacheQueryDao cacheQueryDao = cache.getCacheQueryDao();
try {
waitForRefreshCacheQueueToEmpty();
CacheType type = cacheTypeDao.getByClassName(MockCacheInterface.class.getCanonicalName());
String query = ChebiAnnotator.ID_PREFIX + "1234";
......@@ -450,7 +445,7 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
cache.invalidateByQuery(query, type);
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
assertEquals(1, getErrors().size());
......@@ -488,9 +483,8 @@ public class PermanentDatabaseLevelCacheTest extends AnnotationTestFunctions {
cache.invalidateByQuery(query, type);
while (cache.getRefreshExecutingTasksSize() + cache.getRefreshPendingQueueSize() > 0) {
Thread.sleep(10);
}
permanentDatabaseLevelCache.waitToFinishTasks();
// errors are not caught
assertEquals(0, getErrors().size());
assertEquals(0, getFatals().size());
......
......@@ -6,9 +6,9 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.when;
import java.io.IOException;
......@@ -24,7 +24,8 @@ import org.w3c.dom.Node;
import lcsb.mapviewer.annotation.AnnotationTestFunctions;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.GeneralCacheWithExclusion;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCache;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.annotation.cache.WebPageDownloader;
import lcsb.mapviewer.annotation.cache.XmlSerializer;
......@@ -46,6 +47,9 @@ public class ChEMBLParserTest extends AnnotationTestFunctions {
@Autowired
private GeneralCacheInterface cache;
@Autowired
private PermanentDatabaseLevelCacheInterface permanentDatabaseLevelCache;
@Autowired
private ChEMBLParser chemblParser;
......@@ -740,12 +744,10 @@ public class ChEMBLParserTest extends AnnotationTestFunctions {
String query = "http://google.lu/";
String newRes = "hello";
try {
waitForRefreshCacheQueueToEmpty();
cache.setCachedQuery(query, chemblParser.getCacheType(), newRes);
cache.invalidateByQuery(query, chemblParser.getCacheType());
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
String res = cache.getStringByQuery(query, chemblParser.getCacheType());
......@@ -765,12 +767,10 @@ public class ChEMBLParserTest extends AnnotationTestFunctions {
String query = ChEMBLParser.NAME_PREFIX + "TRIMETHOPRIM";
String newRes = "hello";
try {
waitForRefreshCacheQueueToEmpty();
cache.setCachedQuery(query, chemblParser.getCacheType(), newRes);
cache.invalidateByQuery(query, chemblParser.getCacheType());
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
String res = cache.getStringByQuery(query, chemblParser.getCacheType());
......
......@@ -24,6 +24,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import lcsb.mapviewer.annotation.AnnotationTestFunctions;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.GeneralCacheWithExclusion;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCache;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.annotation.cache.WebPageDownloader;
import lcsb.mapviewer.annotation.data.Chemical;
......@@ -55,6 +57,9 @@ public class ChemicalParserTest extends AnnotationTestFunctions {
@Autowired
private GeneralCacheInterface cache;
@Autowired
private PermanentDatabaseLevelCacheInterface permanentDatabaseLevelCache;
@Test
public void testCreateChemicalListFromDB() throws Exception {
try {
......@@ -229,12 +234,10 @@ public class ChemicalParserTest extends AnnotationTestFunctions {
assertNotNull(chemcials);
assertTrue(chemcials.size() > 0);
waitForRefreshCacheQueueToEmpty();
cache.invalidateByQuery(chemicalParser.getIdentifier(parkinsonDiseaseId, chemcials.get(0).getChemicalId()),
chemicalParser.getCacheType());
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
idsList.clear();
......
......@@ -21,6 +21,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import lcsb.mapviewer.annotation.AnnotationTestFunctions;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCache;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.annotation.cache.WebPageDownloader;
import lcsb.mapviewer.annotation.data.Drug;
......@@ -38,6 +40,9 @@ public class DrugbankHTMLParserTest extends AnnotationTestFunctions {
@Autowired
private DrugbankHTMLParser drugBankHTMLParser;
@Autowired
private PermanentDatabaseLevelCacheInterface permanentDatabaseLevelCache;
@Test
public void test1FindDrug() throws Exception {
try {
......@@ -373,12 +378,10 @@ public class DrugbankHTMLParserTest extends AnnotationTestFunctions {
String query = "http://google.lu/";
String newRes = "hello";
try {
waitForRefreshCacheQueueToEmpty();
cache.setCachedQuery(query, drugBankHTMLParser.getCacheType(), newRes);
cache.invalidateByQuery(query, drugBankHTMLParser.getCacheType());
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
String res = cache.getStringByQuery(query, drugBankHTMLParser.getCacheType());
......@@ -399,12 +402,10 @@ public class DrugbankHTMLParserTest extends AnnotationTestFunctions {
String query = "drug:amantadine";
String newRes = "hello";
try {
waitForRefreshCacheQueueToEmpty();
cache.setCachedQuery(query, drugBankHTMLParser.getCacheType(), newRes);
cache.invalidateByQuery(query, drugBankHTMLParser.getCacheType());
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
String res = cache.getStringByQuery(query, drugBankHTMLParser.getCacheType());
......
......@@ -6,8 +6,8 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.when;
import java.io.IOException;
......@@ -21,6 +21,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import lcsb.mapviewer.annotation.AnnotationTestFunctions;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCache;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.annotation.cache.WebPageDownloader;
import lcsb.mapviewer.annotation.data.MeSH;
......@@ -35,6 +37,9 @@ public class MeSHParserTest extends AnnotationTestFunctions {
@Autowired
MeSHParser meshParser;
@Autowired
private PermanentDatabaseLevelCacheInterface permanentDatabaseLevelCache;
@Autowired
private GeneralCacheInterface cache;
......@@ -173,7 +178,8 @@ public class MeSHParserTest extends AnnotationTestFunctions {
MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D004298");
MeSHParser parserUnderTest = new MeSHParser();
WebPageDownloader webPageDownloader = Mockito.mock(WebPageDownloader.class);
when(webPageDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class))).thenThrow(new IOException());
when(webPageDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
.thenThrow(new IOException());
parserUnderTest.setWebPageDownloader(webPageDownloader);
parserUnderTest.getMeSH(meshID);
......@@ -203,11 +209,9 @@ public class MeSHParserTest extends AnnotationTestFunctions {
MeSH mesh = meshParser.getMeSH(meshID);
assertNotNull(mesh);
waitForRefreshCacheQueueToEmpty();
cache.invalidateByQuery(meshParser.getIdentifier(meshID), meshParser.getCacheType());
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
MeSH mesh2 = meshParser.getMeSH(meshID);
assertNotNull(mesh2);
......@@ -281,7 +285,8 @@ public class MeSHParserTest extends AnnotationTestFunctions {
try {
meshParser.setCache(null);
WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class))).thenThrow(new IOException());
when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
.thenThrow(new IOException());
meshParser.setWebPageDownloader(mockDownloader);
MiriamData meshID = new MiriamData(MiriamType.MESH_2012, "D010300");
......@@ -304,7 +309,8 @@ public class MeSHParserTest extends AnnotationTestFunctions {
WebPageDownloader downloader = meshParser.getWebPageDownloader();
try {
WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class))).thenThrow(new IOException());
when(mockDownloader.getFromNetwork(anyString(), anyString(), nullable(String.class)))
.thenThrow(new IOException());
meshParser.setWebPageDownloader(mockDownloader);
assertEquals(ExternalServiceStatusType.DOWN, meshParser.getServiceStatus().getStatus());
} catch (Exception e) {
......
......@@ -6,7 +6,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.anyString;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
......@@ -23,6 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import lcsb.mapviewer.annotation.AnnotationTestFunctions;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCache;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.annotation.data.MiRNA;
import lcsb.mapviewer.annotation.data.Target;
......@@ -38,6 +40,9 @@ public class MiRNAParserTest extends AnnotationTestFunctions {
@Autowired
MiRNAParser miRNAParser;
@Autowired
private PermanentDatabaseLevelCacheInterface permanentDatabaseLevelCache;
@Before
public void setUp() throws Exception {
}
......@@ -164,11 +169,9 @@ public class MiRNAParserTest extends AnnotationTestFunctions {
assertTrue(!list.isEmpty());
assertEquals(1, list.get(0).getTargets().size());
waitForRefreshCacheQueueToEmpty();
cache.invalidateByQuery(MiRNAParser.MI_RNA_PREFIX + list.get(0).getName(), miRNAParser.getCacheType());
waitForRefreshCacheQueueToEmpty();
permanentDatabaseLevelCache.waitToFinishTasks();
listIds.clear();
listIds.add(list.get(0).getName());
......
......@@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import lcsb.mapviewer.annotation.AnnotationTestFunctions;
import lcsb.mapviewer.annotation.cache.GeneralCacheInterface;
import lcsb.mapviewer.annotation.cache.PermanentDatabaseLevelCacheInterface;
import lcsb.mapviewer.annotation.cache.SourceNotAvailable;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.common.exception.InvalidXmlSchemaException;
......@@ -27,335 +28,336 @@ import lcsb.mapviewer.model.map.MiriamData;
import lcsb.mapviewer.model.map.MiriamType;
public class BiocompendiumAnnotatorTest extends AnnotationTestFunctions {
Logger logger = Logger.getLogger(BiocompendiumAnnotatorTest.class);
MiriamData camk4 = new MiriamData(MiriamType.HGNC_SYMBOL, "CAMK4");
MiriamData slc25a27 = new MiriamData(MiriamType.HGNC_SYMBOL, "SLC25A27");
MiriamData nsmf = new MiriamData(MiriamType.HGNC_SYMBOL, "NSMF");
MiriamData mir449a = new MiriamData(MiriamType.HGNC_SYMBOL, "MIR449A");
@Autowired
private GeneralCacheInterface cache;
@Autowired
private BiocompendiumAnnotator restService;
@Before
public void setUp() {
}
@After
public void tearDown() throws Exception {
}
@Test
@Ignore("Bug 32")
public void testGetAnnotationsForSpecies() throws Exception {
try {
String response = restService.getAnnotation(camk4);
assertNotNull(response);
assertTrue(response.contains("Symbol: CAMK4"));
response = restService.getAnnotation(slc25a27);
assertNotNull(response);
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
@Test
public void testGetters() throws Exception {
try {
assertNotNull(restService.getCommonName());
assertNotNull(restService.getUrl());
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
@Test
public void testGetAnnotationsForInvalidMiriam() throws Exception {
try {
restService.getAnnotation(new MiriamData());
fail("Exception expected");
} catch (InvalidArgumentException e) {
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
@Test
public void testParseXml() throws Exception {
BufferedReader reader;
try {
reader = new BufferedReader(new FileReader("testFiles/annotation/sampleResponse.xml"));
String line = null;
StringBuilder stringBuilder = new StringBuilder();
String ls = System.getProperty("line.separator");
while ((line = reader.readLine()) != null) {
stringBuilder.append(line);
stringBuilder.append(ls);
}
reader.close();
Map<String, String> res = restService.getAnnotationsFromXml(stringBuilder.toString());
assertEquals(2, res.keySet().size());
String response = res.get("CAMK4");
assertNotNull(response);
assertTrue(response.contains("Symbol: CAMK4"));
assertNotNull(res.get("SLC25A27"));
} catch (Exception e) {
e.printStackTrace();
throw e;