Commit 53750fc7 authored by Piotr Gawron's avatar Piotr Gawron
Browse files

Merge branch '389-use-database-migration-framework' into 'master'

Resolve "use database migration framework"

Closes #389

See merge request minerva/core!359
parents e74860da bb3d7c0f
image: node
before_script:
- apt-get update
stages:
- build
- test
......@@ -24,6 +20,7 @@ build_war:
paths:
- minerva.war
script:
- apt-get update
- apt-get install -y curl sudo gnupg git openjdk-8-jdk ant maven
- curl -sL https://deb.nodesource.com/setup_9.x | sudo -E bash -
- sudo apt-get install -y nodejs
......@@ -35,10 +32,13 @@ build_debian_package:
stage: build
only:
- tags
tags:
- privileged
artifacts:
paths:
- "debian/*.deb"
script:
- apt-get update
- apt-get install -y curl sudo gnupg git openjdk-8-jdk ant maven dh-make build-essential lintian devscripts xsltproc fakeroot xsltproc docbook-xsl
- curl -sL https://deb.nodesource.com/setup_9.x | sudo -E bash -
- sudo apt-get install -y nodejs
......@@ -46,3 +46,42 @@ build_debian_package:
- export DEBEMAIL="piotr.gawron@uni.lu"
- ant debian-pkg
test_install_debian_package:
image: debian
stage: test
only:
- tags
# we need to run it in privileged mode due to: https://stackoverflow.com/questions/29683231/tomcat7-fail-to-start-inside-ubuntu-docker-container
tags:
- privileged
services:
- name: piotrgawron/debian-repo
alias: debian-repo
dependencies:
- build_debian_package
script:
- apt-get update
- apt-get install -y ssh software-properties-common dirmngr wget curl
- debian_file=$(ls debian/*.deb)
- echo "PUT $debian_file /docker/incoming" | sftp -o StrictHostKeyChecking=no user@debian-repo
- ssh root@debian-repo /usr/local/sbin/reprepro-import
- apt-add-repository "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main"
- apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
- apt-add-repository "deb http://debian-repo/ unstable main"
- apt-get update
# auto accept oracle license
- echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections
#hacky way of installing minerva on docker - by default rc-invoke is preventing services changes on docker
- mv /usr/sbin/policy-rc.d /usr/sbin/policy-rc.d.backup
- mv Docker/policy-rc.d /usr/sbin/policy-rc.d
- apt-get install -y --allow-unauthenticated minerva
- mv /usr/sbin/policy-rc.d.backup /usr/sbin/policy-rc.d
- service tomcat8 start
#we need to wait a bit for tomcat start
- sleep 15
- wget http://localhost:8080/minerva/
#test if we can login and list projects
- test 200 = $(curl --write-out %{http_code} --silent --output /dev/null -c cookie.txt http://localhost:8080/minerva/api/doLogin)
- test 200 = $(curl --write-out %{http_code} --silent --output /dev/null --cookie cookie.txt http://localhost:8080/minerva/api/projects/)
minerva (12.1.0~alpha.0) experimental; urgency=medium
* Feature: genome browser with gene variants
* Feature: support of all CellDesigner element modifications
* Feature: ldap authentication
* Feature: multicolored anchors for data overlays
* Bug fix: annotations for elements with mutation in name (like "BID (p15)")
weren't provided properly
* Small improvement: annotations from RHEA are handled
* Small improvement: description of a grouped results in improved
* Small improvement: selection of required annotation on map upload improved
-- Piotr Gawron <piotr.gawron@uni.lu> Wed, 25 Jul 2018 10:00:00 +0200
minerva (12.0.1) stable; urgency=medium
* Bug fix: setting default zoom level on submap
* Bug fix: opened submap had different background then current selection
......@@ -8,7 +21,7 @@ minerva (12.0.1) stable; urgency=medium
* Bug fix: map is repositioned when info window is outside of current view in
OpenLayers library
-- Piotr Gawron <piotr.gawron@uni.lu> Wed, 22 Jul 2018 10:00:00 +0200
-- Piotr Gawron <piotr.gawron@uni.lu> Tue, 24 Jul 2018 10:00:00 +0200
minerva (12.0.0) stable; urgency=medium
* Feature: There is alternative OpenLayers map engine available
......
<?xml version="1.0"?>
<project
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>lcsb.mapviewer</groupId>
......@@ -75,8 +76,8 @@
<dependency>
<groupId>org.sbml.jsbml</groupId>
<artifactId>jsbml</artifactId>
<version>1.2</version>
<!-- for now we use lower version of log4j and this one introduce some
<version>1.3.1</version>
<!-- for now we use lower version of log4j and this one introduce some
problem with tomcat loggin -->
<exclusions>
<exclusion>
......@@ -91,6 +92,10 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
......
......@@ -22,7 +22,7 @@ DOC_DIR=doc
TOMCAT_FILE=web/target/web-1.0.war
#Directory with database schema
DB_SCHEMA_DIR=persist/src/db
DB_SCHEMA_DIR=persist/src/main/resources/db/migration
ROOT_DIR=`pwd`
......@@ -46,7 +46,7 @@ cp $DB_SCHEMA_DIR/base.sql $SRC_DIR/db_0.sql
#copy manual documentation
cp $DOC_DIR/description.txt $SRC_DIR/minerva.txt
gzip $SRC_DIR/minerva.txt
gzip -n $SRC_DIR/minerva.txt
#copy configuration of logrotate
cp debian/logrotate/* $SRC_DIR/
......@@ -58,16 +58,18 @@ cp CHANGELOG $ROOT_DIR/debian/template/changelog
# set $versions and $current_version; generate upgrade scripts in SRC_DIR
DBSCRIPT_DEST_DIR=$SRC_DIR
MAX_DB_VERSION_FOR_MIGRTION=12.1.0~alpha
source $dbschemadiff
echo "Current version: " $current_version
echo "Flyway was introduced in version: " $MAX_DB_VERSION_FOR_MIGRTION
# check if the changelog contains info about new version
tmpVar=`cat debian/template/changelog | grep "minerva ($current_version)"|wc -l`
if [ $tmpVar -eq "0" ]
then
echo "Invalid changelog file. Please modify debian/template/changelog to contain information about current version";
exit 1;
e-n xit 1;
fi
......@@ -120,6 +122,7 @@ sed -i "s/__DB_PORT__/$DB_PORT/g" common.sh
sed -i "s/__DB_DATABASE_NAME__/$DB_DATABASE_NAME/g" common.sh
sed -i "s/__DB_USERNAME__/$DB_USERNAME/g" common.sh
sed -i "s/__DB_PASSWORD__/$DB_PASSWORD/g" common.sh
sed -i "s/__MAX_DB_VERSION_FOR_MIGRTION__/$MAX_DB_VERSION_FOR_MIGRTION/g" common.sh
sed -i -e "1r common.sh" debian/postinst
sed -i -e "1r common.sh" debian/postrm
......@@ -132,12 +135,12 @@ sed -i -e "1r common.sh" debian/prerm
#put scripts into $DB_SCRIPT_DIR (it's a bit different than varaiable because it's not
#escaped
echo db_0.sql /usr/share\/minerva/schema >> debian/install
echo db_0_to_$current_version".sql" /usr/share/minerva/schema >> debian/install
echo db_0_to_$MAX_DB_VERSION_FOR_MIGRTION".sql" /usr/share/minerva/schema >> debian/install
for version in ${versions[*]}
do
if [ "$version" != "$current_version" ]
then
echo db_"$version"_to_$current_version".sql" /usr/share/minerva/schema >> debian/install;
echo db_"$version"_to_$MAX_DB_VERSION_FOR_MIGRTION".sql" /usr/share/minerva/schema >> debian/install;
fi
done
......@@ -152,7 +155,8 @@ echo minerva.txt.gz /usr/share/doc/minerva/minerva.txt.gz >> debian/install;
echo minerva /etc/logrotate.d >> debian/install;
#remove some sample file
rm debian/package.doc-base.EX
rm debian/*.EX
rm debian/*.ex
#build debian package
debuild -us -uc
......
......@@ -5,6 +5,7 @@ log(){
LOG_FILE="__LOG_FILE__"
DB_SCRIPT_DIR="__DB_SCRIPT_DIR__"
MAX_DB_VERSION_FOR_MIGRTION="__MAX_DB_VERSION_FOR_MIGRTION__"
#new (current) version of the package
CURRENT_VERSION="__CURRENT_VERSION__"
......
Source: minerva
Priority: optional
Maintainer: Piotr Gawron <piotr.gawron@uni.lu>
Standards-Version: 3.9.6
Standards-Version: 3.9.7
#Vcs-Git: git://anonscm.debian.org/collab-maint/minerva.git
#Vcs-Browser: http://anonscm.debian.org/?p=collab-maint/minerva.git;a=summary
......
......@@ -31,11 +31,15 @@ case "$1" in
#install base version of the framework
exec_sql_file "$DB_SCRIPT_DIR/db_0.sql"
#install patch to current version
exec_sql_file "$DB_SCRIPT_DIR/db_0_to_${CURRENT_VERSION}.sql"
exec_sql_file "$DB_SCRIPT_DIR/db_0_to_${MAX_DB_VERSION_FOR_MIGRTION}.sql"
else
#if we update the package
exec_sql_file "$DB_SCRIPT_DIR/db_${OLD_VERSION}_to_${CURRENT_VERSION}.sql"
if [ -f $DB_SCRIPT_DIR/db_${OLD_VERSION}_to_${MAX_DB_VERSION_FOR_MIGRTION}.sql ]; then
exec_sql_file "$DB_SCRIPT_DIR/db_${OLD_VERSION}_to_${MAX_DB_VERSION_FOR_MIGRTION}.sql"
else
log "INFO: File doesn't exist: $DB_SCRIPT_DIR/db_${OLD_VERSION}_to_${MAX_DB_VERSION_FOR_MIGRTION}.sql"
fi
fi
#print a disclaimer
......
......@@ -4,8 +4,9 @@
# files)
# Precondition: the following variables have to be correctly initialised
# * DB_SCHEMA_DIR set to the directory containing database initialisation and upgrade sql commands
# * DBSCRIPT_DEST_DIR set to the directory that will be populated with database upgrade scripts
# * DB_SCHEMA_DIR set to the directory containing database initialisation and upgrade sql commands
# * DBSCRIPT_DEST_DIR set to the directory that will be populated with database upgrade scripts
# * MAX_DB_VERSION_FOR_MIGRTION set to the max version number that should be used for generating migration scripts; further version will be managed by flyway
# Postcondition:
# * versions is set to the available MINERVA versions (bash array)
......@@ -20,12 +21,22 @@ versions=();
current_version=-1;
#find all available versions
versionCompLte() {
[ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
}
versionCompLt() {
[ "$1" = "$2" ] && return 1 || versionCompLte $1 $2
}
for version in `ls "$DB_SCHEMA_DIR" | sort -V`;
do
if [ -d "$DB_SCHEMA_DIR/$version" ]
then
versions+=($version);
if versionCompLt $version $MAX_DB_VERSION_FOR_MIGRTION
then
versions+=($version);
fi
current_version=$version;
fi
done
......@@ -34,28 +45,24 @@ done
# GENERATE DB schemas and diff files
#--------------------------------------
versionCompLte() {
[ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
}
versionCompLt() {
[ "$1" = "$2" ] && return 1 || versionCompLte $1 $2
}
for version_A in ${versions[*]}
do
touch $DBSCRIPT_DEST_DIR/db_${version_A}_to_${MAX_DB_VERSION_FOR_MIGRTION}.sql
done
#last version (used for generating db diff files)
for version_A in ${versions[*]}
do
#main update file (used to create this version - it's a diff
#between base.sql and current version)
update_file="$DBSCRIPT_DEST_DIR/db_0_to_${current_version}.sql";
#between base.sql and last version not managed by flyway)
update_file="$DBSCRIPT_DEST_DIR/db_0_to_${MAX_DB_VERSION_FOR_MIGRTION}.sql";
#now iterate through all versions
for file in `ls "$DB_SCHEMA_DIR/$version_A" | sort -V`;
do
if [ -f "$DB_SCHEMA_DIR/$version_A/$file" ]
then
#add difference to current db schema diff
#add db schema diff to last version not managed by flyway
printf "\n\n-- UPDATE $version_A/$file\n\n" >> "$update_file";
cat "$DB_SCHEMA_DIR/$version_A/$file" >> "$update_file";
......@@ -67,7 +74,7 @@ do
#after #version_B (so version_A must be later)
if versionCompLt $version_B $version_A
then
upd_file="$DBSCRIPT_DEST_DIR/db_${version_B}_to_${current_version}.sql";
upd_file="$DBSCRIPT_DEST_DIR/db_${version_B}_to_${MAX_DB_VERSION_FOR_MIGRTION}.sql";
printf "\n\n-- UPDATE $version_A/$file\n\n" >> "$upd_file";
cat "$DB_SCHEMA_DIR/$version_A/$file" >> "$upd_file";
fi
......
<project
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>lcsb.mapviewer</groupId>
<artifactId>parent</artifactId>
<version>1.0</version>
</parent>
<artifactId>persist</artifactId>
<name>MapViewer persist</name>
<description>Data Access Interface to the model</description>
<dependencies>
<!-- dependency from the MapViewer model -->
<!-- https://mvnrepository.com/artifact/org.flywaydb/flyway-core -->
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
<version>${flyway.version}</version>
</dependency>
<!-- dependency from the MapViewer model -->
<dependency>
<groupId>lcsb.mapviewer</groupId>
<artifactId>model</artifactId>
<version>1.0</version>
</dependency>
<!-- apache ftp -->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>${commons-net.version}</version>
</dependency>
<!-- apache ftp -->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>${commons-net.version}</version>
</dependency>
<!-- Hibernate -->
<dependency>
<!-- Hibernate -->
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-core</artifactId>
<version>${hibernate.version}</version>
<version>${hibernate.version}</version>
<exclusions>
<exclusion>
<artifactId>xml-apis</artifactId>
<groupId>xml-apis</groupId>
</exclusion>
</exclusions>
</dependency>
<!-- Library excluded from above; it was in conflict with Xerces -->
<dependency>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
<version>${xml-apis.version}</version>
</dependency>
<!-- Log4J -->
</dependency>
<!-- Library excluded from above; it was in conflict with Xerces -->
<dependency>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
<version>${xml-apis.version}</version>
</dependency>
<!-- Log4J -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</dependency>
<!-- Spring -->
<!-- Spring -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>${springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
......@@ -84,28 +90,28 @@
</dependency>
<!-- spring module used for conneting orm (hibernate in our case) -->
<!-- spring module used for conneting orm (hibernate in our case) -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
<version>${springframework.version}</version>
</dependency>
<!-- spring module used for password encoding -->
<!-- spring module used for password encoding -->
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-taglibs</artifactId>
<version>${springframework.security.version}</version>
</dependency>
<!-- spring module used for testing -->
<!-- spring module used for testing -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${springframework.version}</version>
</dependency>
<!-- postgres connector -->
<!-- postgres connector -->
<dependency>
<groupId>postgresql</groupId>
<artifactId>postgresql</artifactId>
......@@ -122,16 +128,16 @@
<groupId>cglib</groupId>
<artifactId>cglib</artifactId>
<version>${cglib.version}</version>
</dependency>
</dependency>
<!-- Gson -->
<!-- Gson -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package lcsb.mapviewer.persist;
import java.sql.Connection;
import java.sql.SQLException;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.flywaydb.core.Flyway;
import org.springframework.jdbc.datasource.init.DatabasePopulator;
import org.springframework.jdbc.datasource.init.ScriptException;
import com.mchange.v2.c3p0.ComboPooledDataSource;
public class CustomDatabasePopulator implements DatabasePopulator {
Logger logger = Logger.getLogger(CustomDatabasePopulator.class);
private DataSource dataSource;
@Override
public void populate(Connection connection) throws SQLException, ScriptException {
ComboPooledDataSource source = (ComboPooledDataSource) dataSource;
String url = source.getJdbcUrl();
String user = source.getUser();
String password = source.getPassword();
Flyway flyway = new Flyway();
flyway.setDataSource(url, user, password);
flyway.setBaselineVersionAsString("12.1.0");
flyway.setBaselineOnMigrate(true);
flyway.migrate();
}
public DataSource getDataSource() {
return dataSource;
}
public void setDataSource(DataSource dataSource) {
this.dataSource = dataSource;
}
}
......@@ -9,12 +9,17 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import org.apache.log4j.Logger;
import org.flywaydb.core.Flyway;
import org.hibernate.SQLQuery;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.internal.SessionFactoryImpl;
import org.hibernate.jmx.StatisticsService;
import org.hibernate.service.jdbc.connections.internal.DatasourceConnectionProviderImpl;
import org.springframework.beans.factory.annotation.Autowired;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lcsb.mapviewer.common.exception.InvalidStateException;
/**
......@@ -33,275 +38,275 @@ import lcsb.mapviewer.common.exception.InvalidStateException;
*/
@SuppressWarnings("deprecation")
public class DbUtils extends Observable {
/**
* Default class logger.
*/
private static Logger logger = Logger.getLogger(DbUtils.class);
/**
* Hibernate session factory.
*/
@Autowired
private SessionFactory sessionFactory;
/**
* Default class logger.
*/
private static Logger logger = Logger.getLogger(DbUtils.class);
/**
* Hibernate session factory.
*/
@Autowired
private SessionFactory sessionFactory;
/**
* Service used for connection statistics.
*/
private StatisticsService statisticsService;
/**
* Service used for connection statistics.
*/
private StatisticsService statisticsService;
/**
* This flag indicates if the object was initialized or not.
*
* @see #init()
*/
private boolean initialized = false;
/**
* Bean used for statistics.
*/
private MBeanServer mbeanServer;
/**
* This flag indicates if the object was initialized or not.
*
* @see #init()
*/
private boolean initialized = false;
/**
* Static map containing opened custom sessions. This object is also used for
* synchronization between threads when accessing informations about sessions.
*/
private static Map<Long, Session> sessionForThread = new HashMap<Long, Session>();
/**
* Bean used for statistics.
*/
private MBeanServer mbeanServer;
/**
* This determines if every add/update/delete operation should be flushed (for
* specific {@link Thread}). There are few drawbacks of this approach:
* <ol>
* <li>when autoflushing is set to false, then the data consistency could be
* broken (even in the same thread/transaction)</li>,
* <li>we have to automatically take care of the flushing, therefore there
* might be some problems with long transactions.</li>
* </ol>
*/
private static Map<Long, Boolean> autoFlushForThread = new HashMap<>();
/**
* Static map containing opened custom sessions. This object is also used for
* synchronization between threads when accessing informations about sessions.
*/
private static Map<Long, Session> sessionForThread = new HashMap<Long, Session>();
/**
* This method initialize services responsible for statistics.
*/
private void init() {
if (!initialized) {
if (SpringApplicationContext.getApplicationContext().containsBean("mbeanServer")) {
logger.info("Hibernate statistics turned on");
mbeanServer = SpringApplicationContext.getApplicationContext().getBean(MBeanServer.class);
statisticsService = new StatisticsService();
statisticsService.setSessionFactory(sessionFactory);