From 7c1b9aedd528f70eeacbd846666e2f663198d808 Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Fri, 14 Nov 2014 11:06:59 +0100 Subject: [PATCH] Setup a docker container that holds a webdav instance to test files_external document docker parameter and use random host port fix typo copy autotest.sh to autotest-external.sh adds ability to add start* and stop* scripts in env in external tests run files_external WebDAV tests against ownCloud instance introduce executor number to be able to shut down the correct docker container fetch docker images in advance - this also fetches latest versions of the docker images add second argument to autotest-external.sh which can specify a single test to run print out the explicit test run change naming schema of files_external setup scripts --- .gitignore | 1 + .../tests/{ => backends}/amazons3.php | 0 .../tests/{ => backends}/dropbox.php | 0 .../tests/{ => backends}/ftp.php | 0 .../tests/{ => backends}/google.php | 0 .../tests/{ => backends}/owncloud.php | 0 .../tests/{ => backends}/sftp.php | 0 .../tests/{ => backends}/smb.php | 0 .../tests/{ => backends}/swift.php | 0 .../tests/{ => backends}/webdav.php | 12 +- .../tests/env/start-webdav-ownCloud.sh | 82 +++++ .../tests/env/stop-webdav-ownCloud.sh | 41 +++ autotest-external.sh | 308 ++++++++++++++++++ tests/phpunit-autotest-external.xml | 36 ++ 14 files changed, 474 insertions(+), 6 deletions(-) rename apps/files_external/tests/{ => backends}/amazons3.php (100%) rename apps/files_external/tests/{ => backends}/dropbox.php (100%) rename apps/files_external/tests/{ => backends}/ftp.php (100%) rename apps/files_external/tests/{ => backends}/google.php (100%) rename apps/files_external/tests/{ => backends}/owncloud.php (100%) rename apps/files_external/tests/{ => backends}/sftp.php (100%) rename apps/files_external/tests/{ => backends}/smb.php (100%) rename apps/files_external/tests/{ => backends}/swift.php (100%) rename apps/files_external/tests/{ => backends}/webdav.php (56%) create mode 100755 apps/files_external/tests/env/start-webdav-ownCloud.sh create mode 100755 apps/files_external/tests/env/stop-webdav-ownCloud.sh create mode 100755 autotest-external.sh create mode 100644 tests/phpunit-autotest-external.xml diff --git a/.gitignore b/.gitignore index 93bacf9b07..9d8c40b34a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ /apps/files_external/3rdparty/irodsphp/prods/test /apps/files_external/3rdparty/irodsphp/prods/tutorials /apps/files_external/3rdparty/irodsphp/prods/test* +/apps/files_external/tests/config.*.php diff --git a/apps/files_external/tests/amazons3.php b/apps/files_external/tests/backends/amazons3.php similarity index 100% rename from apps/files_external/tests/amazons3.php rename to apps/files_external/tests/backends/amazons3.php diff --git a/apps/files_external/tests/dropbox.php b/apps/files_external/tests/backends/dropbox.php similarity index 100% rename from apps/files_external/tests/dropbox.php rename to apps/files_external/tests/backends/dropbox.php diff --git a/apps/files_external/tests/ftp.php b/apps/files_external/tests/backends/ftp.php similarity index 100% rename from apps/files_external/tests/ftp.php rename to apps/files_external/tests/backends/ftp.php diff --git a/apps/files_external/tests/google.php b/apps/files_external/tests/backends/google.php similarity index 100% rename from apps/files_external/tests/google.php rename to apps/files_external/tests/backends/google.php diff --git a/apps/files_external/tests/owncloud.php b/apps/files_external/tests/backends/owncloud.php similarity index 100% rename from apps/files_external/tests/owncloud.php rename to apps/files_external/tests/backends/owncloud.php diff --git a/apps/files_external/tests/sftp.php b/apps/files_external/tests/backends/sftp.php similarity index 100% rename from apps/files_external/tests/sftp.php rename to apps/files_external/tests/backends/sftp.php diff --git a/apps/files_external/tests/smb.php b/apps/files_external/tests/backends/smb.php similarity index 100% rename from apps/files_external/tests/smb.php rename to apps/files_external/tests/backends/smb.php diff --git a/apps/files_external/tests/swift.php b/apps/files_external/tests/backends/swift.php similarity index 100% rename from apps/files_external/tests/swift.php rename to apps/files_external/tests/backends/swift.php diff --git a/apps/files_external/tests/webdav.php b/apps/files_external/tests/backends/webdav.php similarity index 56% rename from apps/files_external/tests/webdav.php rename to apps/files_external/tests/backends/webdav.php index 5f53568b91..c390612810 100644 --- a/apps/files_external/tests/webdav.php +++ b/apps/files_external/tests/backends/webdav.php @@ -16,15 +16,15 @@ class DAV extends Storage { parent::setUp(); $id = $this->getUniqueID(); - $this->config = include('files_external/tests/config.php'); - if ( ! is_array($this->config) or ! isset($this->config['webdav']) or ! $this->config['webdav']['run']) { + $config = include('files_external/tests/config.webdav.php'); + if ( ! is_array($config) or !$config['run']) { $this->markTestSkipped('WebDAV backend not configured'); } - if (isset($this->config['webdav']['wait'])) { - $this->waitDelay = $this->config['webdav']['wait']; + if (isset($config['wait'])) { + $this->waitDelay = $config['wait']; } - $this->config['webdav']['root'] .= '/' . $id; //make sure we have an new empty folder to work in - $this->instance = new \OC\Files\Storage\DAV($this->config['webdav']); + $config['root'] .= '/' . $id; //make sure we have an new empty folder to work in + $this->instance = new \OC\Files\Storage\DAV($config); $this->instance->mkdir('/'); } diff --git a/apps/files_external/tests/env/start-webdav-ownCloud.sh b/apps/files_external/tests/env/start-webdav-ownCloud.sh new file mode 100755 index 0000000000..c7267cff34 --- /dev/null +++ b/apps/files_external/tests/env/start-webdav-ownCloud.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# +# ownCloud +# +# This script start a docker container to test the files_external tests +# against. It will also change the files_external config to use the docker +# container as testing environment. This is reverted in the stop step. +# +# If the environment variable RUN_DOCKER_MYSQL is set the ownCloud will +# be set up using MySQL instead of SQLite. +# +# Set environment variable DEBUG to print config file +# +# @author Morris Jobke +# @copyright 2014 Morris Jobke +# + +if ! command -v docker >/dev/null 2>&1; then + echo "No docker executable found - skipped docker setup" + exit 0; +fi + +echo "Docker executable found - setup docker" + +echo "Fetch recent morrisjobke/owncloud docker image" +docker pull morrisjobke/owncloud + +# retrieve current folder to place the config in the parent folder +thisFolder=`echo $0 | replace "env/start-webdav-ownCloud.sh" ""` + +if [ -n "$RUN_DOCKER_MYSQL" ]; then + echo "Fetch recent mysql docker image" + docker pull mysql + + echo "Setup MySQL ..." + # user/password will be read by ENV variables in owncloud container (they are generated by docker) + databaseContainer=`docker run -e MYSQL_ROOT_PASSWORD=mysupersecretpassword -d mysql` + containerName=`docker inspect $databaseContainer | grep Name | grep _ | cut -d \" -f 4 | cut -d / -f 2` + + parameter="--link $containerName:db" +fi + +container=`docker run -P $parameter -d -e ADMINLOGIN=test -e ADMINPWD=test morrisjobke/owncloud` + +# TODO find a way to determine the successful initialization inside the docker container +echo "Waiting 30 seconds for ownCloud initialization ... " +sleep 30 + +# get mapped port on host for internal port 80 - output is IP:PORT - we need to extract the port with 'cut' +port=`docker port $container 80 | cut -f 2 -d :` + + +cat > $thisFolder/config.webdav.php <true, + 'host'=>'localhost:$port/owncloud/remote.php/webdav/', + 'user'=>'test', + 'password'=>'test', + 'root'=>'', + // wait delay in seconds after write operations + // (only in tests) + // set to higher value for lighttpd webdav + 'wait'=> 0 +); + +DELIM + +echo "ownCloud container: $container" + +# put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host) +echo $container >> $thisFolder/dockerContainerOwnCloud.$EXECUTOR_NUMBER.webdav + +if [ -n "$databaseContainer" ]; then + echo "Database container: $databaseContainer" + echo $databaseContainer >> $thisFolder/dockerContainerOwnCloud.$EXECUTOR_NUMBER.webdav +fi + +if [ -n "$DEBUG" ]; then + echo $thisFolder/config.webdav.php +fi diff --git a/apps/files_external/tests/env/stop-webdav-ownCloud.sh b/apps/files_external/tests/env/stop-webdav-ownCloud.sh new file mode 100755 index 0000000000..2f06eadcf7 --- /dev/null +++ b/apps/files_external/tests/env/stop-webdav-ownCloud.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# ownCloud +# +# This script stops the docker container the files_external tests were run +# against. It will also revert the config changes done in start step. +# +# @author Morris Jobke +# @copyright 2014 Morris Jobke +# + +if ! command -v docker >/dev/null 2>&1; then + echo "No docker executable found - skipped docker stop" + exit 0; +fi + +echo "Docker executable found - stop and remove docker containers" + +# retrieve current folder to remove the config from the parent folder +thisFolder=`echo $0 | replace "env/stop-webdav-ownCloud.sh" ""` + +echo "DEBUG" + +netstat -tlpen + +echo "CONFIG:" + +cat $thisFolder/config.webdav.php +cat $thisFolder/dockerContainerOwnCloud.$EXECUTOR_NUMBER.webdav + +# stopping and removing docker containers +for container in `cat $thisFolder/dockerContainerOwnCloud.$EXECUTOR_NUMBER.webdav`; do + echo "Stopping and removing docker container $container" + # kills running container and removes it + docker rm -f $container +done; + +# cleanup +rm $thisFolder/config.webdav.php +rm $thisFolder/dockerContainerOwnCloud.$EXECUTOR_NUMBER.webdav + diff --git a/autotest-external.sh b/autotest-external.sh new file mode 100755 index 0000000000..761477a4c9 --- /dev/null +++ b/autotest-external.sh @@ -0,0 +1,308 @@ +#!/bin/bash +# +# ownCloud +# +# @author Thomas Müller +# @author Morris Jobke +# @copyright 2012, 2013 Thomas Müller thomas.mueller@tmit.eu +# @copyright 2014 Morris Jobke hey@morrisjobke.de +# + +#$EXECUTOR_NUMBER is set by Jenkins and allows us to run autotest in parallel +DATABASENAME=oc_autotest$EXECUTOR_NUMBER +DATABASEUSER=oc_autotest$EXECUTOR_NUMBER +ADMINLOGIN=admin$EXECUTOR_NUMBER +BASEDIR=$PWD + +DBCONFIGS="sqlite mysql pgsql oci" +PHPUNIT=$(which phpunit) + +function print_syntax { + echo -e "Syntax: ./autotest-external.sh [dbconfigname] [startfile]\n" >&2 + echo -e "\t\"dbconfigname\" can be one of: $DBCONFIGS" >&2 + echo -e "\t\"startfile\" is the name of a start file inside the env/ folder in the files_external tests" >&2 + echo -e "\nExample: ./autotest.sh sqlite webdav-ownCloud" >&2 + echo "will run the external suite from \"apps/files_external/tests/env/start-webdav-ownCloud.sh\"" >&2 + echo -e "\nIf no arguments are specified, all available external backends will be run with all database configs" >&2 +} + +if ! [ -x "$PHPUNIT" ]; then + echo "phpunit executable not found, please install phpunit version >= 3.7" >&2 + exit 3 +fi + +PHPUNIT_VERSION=$("$PHPUNIT" --version | cut -d" " -f2) +PHPUNIT_MAJOR_VERSION=$(echo $PHPUNIT_VERSION | cut -d"." -f1) +PHPUNIT_MINOR_VERSION=$(echo $PHPUNIT_VERSION | cut -d"." -f2) + +if ! [ $PHPUNIT_MAJOR_VERSION -gt 3 -o \( $PHPUNIT_MAJOR_VERSION -eq 3 -a $PHPUNIT_MINOR_VERSION -ge 7 \) ]; then + echo "phpunit version >= 3.7 required. Version found: $PHPUNIT_VERSION" >&2 + exit 4 +fi + +if ! [ \( -w config -a ! -f config/config.php \) -o \( -f config/config.php -a -w config/config.php \) ]; then + echo "Please enable write permissions on config and config/config.php" >&2 + exit 1 +fi + +if [ "$1" ]; then + FOUND=0 + for DBCONFIG in $DBCONFIGS; do + if [ "$1" = $DBCONFIG ]; then + FOUND=1 + break + fi + done + if [ $FOUND = 0 ]; then + echo -e "Unknown database config name \"$1\"\n" >&2 + print_syntax + exit 2 + fi +fi + +# Back up existing (dev) config if one exists +if [ -f config/config.php ]; then + mv config/config.php config/config-autotest-backup.php +fi + +function restore_config { + # Restore existing config + if [ -f config/config-autotest-backup.php ]; then + mv config/config-autotest-backup.php config/config.php + fi +} + +# restore config on exit, even when killed +trap restore_config SIGINT SIGTERM + +# use tmpfs for datadir - should speedup unit test execution +if [ -d /dev/shm ]; then + DATADIR=/dev/shm/data-autotest$EXECUTOR_NUMBER +else + DATADIR=$BASEDIR/data-autotest +fi + +echo "Using database $DATABASENAME" + +# create autoconfig for sqlite, mysql and postgresql +cat > ./tests/autoconfig-sqlite.php < false, + 'dbtype' => 'sqlite', + 'dbtableprefix' => 'oc_', + 'adminlogin' => '$ADMINLOGIN', + 'adminpass' => 'admin', + 'directory' => '$DATADIR', +); +DELIM + +cat > ./tests/autoconfig-mysql.php < false, + 'dbtype' => 'mysql', + 'dbtableprefix' => 'oc_', + 'adminlogin' => '$ADMINLOGIN', + 'adminpass' => 'admin', + 'directory' => '$DATADIR', + 'dbuser' => '$DATABASEUSER', + 'dbname' => '$DATABASENAME', + 'dbhost' => 'localhost', + 'dbpass' => 'owncloud', +); +DELIM + +cat > ./tests/autoconfig-pgsql.php < false, + 'dbtype' => 'pgsql', + 'dbtableprefix' => 'oc_', + 'adminlogin' => '$ADMINLOGIN', + 'adminpass' => 'admin', + 'directory' => '$DATADIR', + 'dbuser' => '$DATABASEUSER', + 'dbname' => '$DATABASENAME', + 'dbhost' => 'localhost', + 'dbpass' => 'owncloud', +); +DELIM + +cat > ./tests/autoconfig-oci.php < false, + 'dbtype' => 'oci', + 'dbtableprefix' => 'oc_', + 'adminlogin' => '$ADMINLOGIN', + 'adminpass' => 'admin', + 'directory' => '$DATADIR', + 'dbuser' => '$DATABASENAME', + 'dbname' => 'XE', + 'dbhost' => 'localhost', + 'dbpass' => 'owncloud', +); +DELIM + +function execute_tests { + echo "Setup environment for $1 testing ..." + # back to root folder + cd "$BASEDIR" + + # revert changes to tests/data + git checkout tests/data + + # reset data directory + rm -rf "$DATADIR" + mkdir "$DATADIR" + + # remove the old config file + #rm -rf config/config.php + cp tests/preseed-config.php config/config.php + + # drop database + if [ "$1" == "mysql" ] ; then + mysql -u $DATABASEUSER -powncloud -e "DROP DATABASE IF EXISTS $DATABASENAME" || true + fi + if [ "$1" == "pgsql" ] ; then + dropdb -U $DATABASEUSER $DATABASENAME || true + fi + if [ "$1" == "oci" ] ; then + echo "drop the database" + sqlplus -s -l / as sysdba < + + + ../apps/files_external/tests + + ../apps/files_external/tests/backends/ + + + + + + ../lib/private/files/storage/dav.php + ../apps/files_external + + ../apps/files_external/l10n + ../apps/files_external/3rdparty + ../apps/files_external/tests + + + + + +