Merge pull request #14077 from owncloud/autotest-external-swift

[files_external] swift tests

apps/files_external/tests/backends/swift.php

@@ -32,28 +32,35 @@ class Swift extends Storage {
 	protected function setUp() {
 		parent::setUp();
 
-		$this->config = include('files_external/tests/config.php');
+		$this->config = include('files_external/tests/config.swift.php');
-		if (!is_array($this->config) or !isset($this->config['swift'])
+		if (!is_array($this->config) or !$this->config['run']) {
-                    or !$this->config['swift']['run']) {
 			$this->markTestSkipped('OpenStack Object Storage backend not configured');
 		}
-		$this->instance = new \OC\Files\Storage\Swift($this->config['swift']);
+		$this->instance = new \OC\Files\Storage\Swift($this->config);
 	}
 
 	protected function tearDown() {
 		if ($this->instance) {
-			$connection = $this->instance->getConnection();
+			try {
-			$container = $connection->getContainer($this->config['swift']['bucket']);
+				$connection = $this->instance->getConnection();
+				$container = $connection->getContainer($this->config['bucket']);
 
-			$objects = $container->objectList();
+				$objects = $container->objectList();
-			while($object = $objects->next()) {
+				while($object = $objects->next()) {
-				$object->setName(str_replace('#','%23',$object->getName()));
+					$object->setName(str_replace('#','%23',$object->getName()));
-				$object->delete();
+					$object->delete();
+				}
+
+				$container->delete();
+			} catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
+				// container didn't exist, so we don't need to delete it
 			}
-
-			$container->delete();
 		}
 
 		parent::tearDown();
 	}
+
+	public function testStat() {
+		$this->markTestSkipped('Swift doesn\'t update the parents folder mtime');
+	}
 }

apps/files_external/tests/env/start-amazons3-ceph.sh

@@ -49,9 +49,21 @@ echo "${docker_image} container: $container"
 # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host)
 echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.amazons3
 
-# TODO find a way to determine the successful initialization inside the docker container
+echo -n "Waiting for ceph initialization"
-echo "Waiting 20 seconds for ceph initialization ... "
+starttime=$(date +%s)
-sleep 20
+# support for GNU netcat and BSD netcat
+while ! (nc -c -w 1 ${host} ${port} </dev/null >&/dev/null \
+    || nc -w 1 ${host} ${port} </dev/null >&/dev/null); do
+    sleep 1
+    echo -n '.'
+    if (( $(date +%s) > starttime + 60 )); then
+	echo
+	echo "[ERROR] Waited 60 seconds, no response" >&2
+	exit 1
+    fi
+done
+echo
+sleep 1
 
 echo "Create ceph user"
 docker exec $container radosgw-admin user create \

apps/files_external/tests/env/start-ftp-morrisjobke.sh

@@ -54,12 +54,25 @@ echo "ftp container: $container"
 # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host)
 echo $container >> $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.ftp
 
+echo -n "Waiting for ftp initialization"
+starttime=$(date +%s)
+# support for GNU netcat and BSD netcat
+while ! (nc -c -w 1 ${host} 21 </dev/null >&/dev/null \
+    || nc -w 1 ${host} 21 </dev/null >&/dev/null); do
+    sleep 1
+    echo -n '.'
+    if (( $(date +%s) > starttime + 60 )); then
+	echo
+	echo "[ERROR] Waited 60 seconds, no response" >&2
+	exit 1
+    fi
+done
+echo
+sleep 1
+
 if [ -n "$DEBUG" ]; then
     cat $thisFolder/config.ftp.php
     cat $thisFolder/dockerContainerMorrisJobke.$EXECUTOR_NUMBER.ftp
 fi
 
-# TODO find a way to determine the successful initialization inside the docker container
-echo "Waiting 5 seconds for ftp initialization ... "
-sleep 5
 

apps/files_external/tests/env/start-sftp-atmoz.sh

@@ -54,15 +54,27 @@ echo "sftp container: $container"
 # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host)
 echo $container >> $thisFolder/dockerContainerAtmoz.$EXECUTOR_NUMBER.sftp
 
+echo -n "Waiting for sftp initialization"
+starttime=$(date +%s)
+# support for GNU netcat and BSD netcat
+while ! (nc -c -w 1 ${host} 22 </dev/null >&/dev/null \
+    || nc -w 1 ${host} 22 </dev/null >&/dev/null); do
+    sleep 1
+    echo -n '.'
+    if (( $(date +%s) > starttime + 60 )); then
+	echo
+	echo "[ERROR] Waited 60 seconds, no response" >&2
+	exit 1
+    fi
+done
+echo
+sleep 1
+
 if [ -n "$DEBUG" ]; then
     cat $thisFolder/config.sftp.php
     cat $thisFolder/dockerContainerAtmoz.$EXECUTOR_NUMBER.sftp
 fi
 
-# TODO find a way to determine the successful initialization inside the docker container
-echo "Waiting 5 seconds for sftp initialization ... "
-sleep 5
-
 # create folder "upload" with correct permissions
 docker exec $container bash -c "mkdir /home/$user/upload && chown $user:users /home/$user/upload"
 

apps/files_external/tests/env/start-smb-silvershell.sh

@@ -52,12 +52,25 @@ echo "samba container: $container"
 # put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host)
 echo $container >> $thisFolder/dockerContainerSilvershell.$EXECUTOR_NUMBER.smb
 
+echo -n "Waiting for samba initialization"
+starttime=$(date +%s)
+# support for GNU netcat and BSD netcat
+while ! (nc -c -w 1 ${host} 445 </dev/null >&/dev/null \
+    || nc -w 1 ${host} 445 </dev/null >&/dev/null); do
+    sleep 1
+    echo -n '.'
+    if (( $(date +%s) > starttime + 60 )); then
+	echo
+	echo "[ERROR] Waited 60 seconds, no response" >&2
+	exit 1
+    fi
+done
+echo
+sleep 1
+
 if [ -n "$DEBUG" ]; then
     cat $thisFolder/config.smb.php
     cat $thisFolder/dockerContainerSilvershell.$EXECUTOR_NUMBER.smb
 fi
 
-# TODO find a way to determine the successful initialization inside the docker container
-echo "Waiting 5 seconds for smbd initialization ... "
-sleep 5
 

apps/files_external/tests/env/start-smb-windows.sh

@@ -19,6 +19,12 @@ user=smb-test
 password=!owncloud123
 host=WIN-9GTFAS08C15
 
+if ! (nc -c -w 1 ${host} 445 </dev/null >&/dev/null \
+    || nc -w 1 ${host} 445 </dev/null >&/dev/null); then
+    echo "[ERROR] Server not reachable" >&2
+    exit 1
+fi
+
 cat > $thisFolder/config.smb.php <<DELIM
 <?php
 

apps/files_external/tests/env/start-swift-ceph.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# ownCloud
+#
+# This script start a docker container to test the files_external tests
+# against. It will also change the files_external config to use the docker
+# container as testing environment. This is reverted in the stop step.W
+#
+# Set environment variable DEBUG to print config file
+#
+# @author Morris Jobke
+# @author Robin McCorkell
+# @copyright 2015 ownCloud
+
+if ! command -v docker >/dev/null 2>&1; then
+    echo "No docker executable found - skipped docker setup"
+    exit 0;
+fi
+
+echo "Docker executable found - setup docker"
+
+docker_image=xenopathic/ceph-keystone
+
+echo "Fetch recent ${docker_image} docker image"
+docker pull ${docker_image}
+
+# retrieve current folder to place the config in the parent folder
+thisFolder=`echo $0 | replace "env/start-swift-ceph.sh" ""`
+
+if [ -z "$thisFolder" ]; then
+    thisFolder="."
+fi;
+
+port=5001
+
+user=test
+pass=testing
+tenant=testenant
+region=testregion
+service=testceph
+
+container=`docker run -d \
+    -e KEYSTONE_PUBLIC_PORT=${port} \
+    -e KEYSTONE_ADMIN_USER=${user} \
+    -e KEYSTONE_ADMIN_PASS=${pass} \
+    -e KEYSTONE_ADMIN_TENANT=${tenant} \
+    -e KEYSTONE_ENDPOINT_REGION=${region} \
+    -e KEYSTONE_SERVICE=${service} \
+    ${docker_image}`
+
+host=`docker inspect --format="{{.NetworkSettings.IPAddress}}" $container`
+
+
+echo "${docker_image} container: $container"
+
+# put container IDs into a file to drop them after the test run (keep in mind that multiple tests run in parallel on the same host)
+echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
+
+echo -n "Waiting for ceph initialization"
+starttime=$(date +%s)
+# support for GNU netcat and BSD netcat
+while ! (nc -c -w 1 ${host} 80 </dev/null >&/dev/null \
+    || nc -w 1 ${host} 80 </dev/null >&/dev/null); do
+    sleep 1
+    echo -n '.'
+    if (( $(date +%s) > starttime + 60 )); then
+	echo
+	echo "[ERROR] Waited 60 seconds, no response" >&2
+	exit 1
+    fi
+done
+echo
+sleep 1
+
+cat > $thisFolder/config.swift.php <<DELIM
+<?php
+
+return array(
+    'run'=>true,
+    'url'=>'http://$host:$port/v2.0',
+    'user'=>'$user',
+    'tenant'=>'$tenant',
+    'password'=>'$pass',
+    'service_name'=>'$service',
+    'bucket'=>'swift',
+    'region' => '$region',
+);
+
+DELIM
+
+if [ -n "$DEBUG" ]; then
+    cat $thisFolder/config.swift.php
+    cat $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
+fi

apps/files_external/tests/env/start-webdav-ownCloud.sh

@@ -46,20 +46,30 @@ fi
 
 container=`docker run -P $parameter -d -e ADMINLOGIN=test -e ADMINPWD=test morrisjobke/owncloud`
 
-# TODO find a way to determine the successful initialization inside the docker container
+host=`docker inspect $container | grep IPAddress | cut -d '"' -f 4`
-echo "Waiting 30 seconds for ownCloud initialization ... "
-sleep 30
-
-# get mapped port on host for internal port 80 - output is IP:PORT - we need to extract the port with 'cut'
-port=`docker port $container 80 | cut -f 2 -d :`
 
+echo -n "Waiting for ownCloud initialization"
+starttime=$(date +%s)
+# support for GNU netcat and BSD netcat
+while ! (nc -c -w 1 ${host} 80 </dev/null >&/dev/null \
+    || nc -w 1 ${host} 80 </dev/null >&/dev/null); do
+    sleep 1
+    echo -n '.'
+    if (( $(date +%s) > starttime + 60 )); then
+	echo
+	echo "[ERROR] Waited 60 seconds, no response" >&2
+	exit 1
+    fi
+done
+echo
+sleep 1
 
 cat > $thisFolder/config.webdav.php <<DELIM
 <?php
 
 return array(
     'run'=>true,
-    'host'=>'localhost:$port/owncloud/remote.php/webdav/',
+    'host'=>'${host}:80/owncloud/remote.php/webdav/',
     'user'=>'test',
     'password'=>'test',
     'root'=>'',

apps/files_external/tests/env/stop-swift-ceph.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# ownCloud
+#
+# This script stops the docker container the files_external tests were run
+# against. It will also revert the config changes done in start step.
+#
+# @author Morris Jobke
+# @author Robin McCorkell
+# @copyright 2015 ownCloud
+
+if ! command -v docker >/dev/null 2>&1; then
+    echo "No docker executable found - skipped docker stop"
+    exit 0;
+fi
+
+echo "Docker executable found - stop and remove docker containers"
+
+# retrieve current folder to remove the config from the parent folder
+thisFolder=`echo $0 | replace "env/stop-swift-ceph.sh" ""`
+
+if [ -z "$thisFolder" ]; then
+    thisFolder="."
+fi;
+
+# stopping and removing docker containers
+for container in `cat $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift`; do
+    echo "Stopping and removing docker container $container"
+    # kills running container and removes it
+    docker rm -f $container
+done;
+
+# cleanup
+rm $thisFolder/config.swift.php
+rm $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
+

autotest-external.sh

@@ -175,57 +175,56 @@ EOF
 	fi
 
 	if [ -n "$2" -a "$2" == "common-tests" ]; then
-	    return;
+		return;
 	fi
 
-    FILES_EXTERNAL_BACKEND_PATH=../apps/files_external/tests/backends
+	FILES_EXTERNAL_BACKEND_PATH=../apps/files_external/tests/backends
-    FILES_EXTERNAL_BACKEND_ENV_PATH=../apps/files_external/tests/env
+	FILES_EXTERNAL_BACKEND_ENV_PATH=../apps/files_external/tests/env
 
 	for startFile in `ls -1 $FILES_EXTERNAL_BACKEND_ENV_PATH | grep start`; do
-	    name=`echo $startFile | sed 's/start-//' | sed 's/\.sh//'`
+		name=`echo $startFile | sed 's/start-//' | sed 's/\.sh//'`
 
-	    if [ -n "$2" -a "$2" != "$name" ]; then
+		if [ -n "$2" -a "$2" != "$name" ]; then
-	        echo "skip: $startFile"
+			echo "skip: $startFile"
-	        continue;
+			continue;
-	    fi
+		fi
 
-	    echo "start: $startFile"
+		echo "start: $startFile"
-	    echo "name: $name"
+		echo "name: $name"
 
-	    # execute start file
+		# execute start file
-	    ./$FILES_EXTERNAL_BACKEND_ENV_PATH/$startFile
+		if ./$FILES_EXTERNAL_BACKEND_ENV_PATH/$startFile; then
+			# getting backend to test from filename
+			# it's the part between the dots startSomething.TestToRun.sh
+			testToRun=`echo $startFile | cut -d '-' -f 2`
 
-	    # getting backend to test from filename
+			# run the specific test
-	    # it's the part between the dots startSomething.TestToRun.sh
+			if [ -z "$NOCOVERAGE" ]; then
-	    testToRun=`echo $startFile | cut -d '-' -f 2`
+				rm -rf "coverage-external-html-$1-$name"
+				mkdir "coverage-external-html-$1-$name"
+				"$PHPUNIT" --configuration phpunit-autotest-external.xml --log-junit "autotest-external-results-$1-$name.xml" --coverage-clover "autotest-external-clover-$1-$name.xml" --coverage-html "coverage-external-html-$1-$name" "$FILES_EXTERNAL_BACKEND_PATH/$testToRun.php"
+				RESULT=$?
+			else
+				echo "No coverage"
+				"$PHPUNIT" --configuration phpunit-autotest-external.xml --log-junit "autotest-external-results-$1-$name.xml" "$FILES_EXTERNAL_BACKEND_PATH/$testToRun.php"
+				RESULT=$?
+			fi
+		fi
 
-        # run the specific test
+		# calculate stop file
-        if [ -z "$NOCOVERAGE" ]; then
+		stopFile=`echo "$startFile" | sed 's/start/stop/'`
-            rm -rf "coverage-external-html-$1-$name"
+		echo "stop: $stopFile"
-            mkdir "coverage-external-html-$1-$name"
+		if [ -f $FILES_EXTERNAL_BACKEND_ENV_PATH/$stopFile ]; then
-            "$PHPUNIT" --configuration phpunit-autotest-external.xml --log-junit "autotest-external-results-$1-$name.xml" --coverage-clover "autotest-external-clover-$1-$name.xml" --coverage-html "coverage-external-html-$1-$name" "$FILES_EXTERNAL_BACKEND_PATH/$testToRun.php"
+			# execute stop file if existant
-            RESULT=$?
+			./$FILES_EXTERNAL_BACKEND_ENV_PATH/$stopFile
-        else
+		fi
-            echo "No coverage"
-            "$PHPUNIT" --configuration phpunit-autotest-external.xml --log-junit "autotest-external-results-$1-$name.xml" "$FILES_EXTERNAL_BACKEND_PATH/$testToRun.php"
-            RESULT=$?
-        fi
-
-	    # calculate stop file
-	    stopFile=`echo "$startFile" | sed 's/start/stop/'`
-	    echo "stop: $stopFile"
-	    if [ -f $FILES_EXTERNAL_BACKEND_ENV_PATH/$stopFile ]; then
-	        # execute stop file if existant
-	        ./$FILES_EXTERNAL_BACKEND_ENV_PATH/$stopFile
-	    fi
 	done;
 }
 
 #
 # start test execution
 #
-if [ -z "$1" ]
+if [ -z "$1" ]; then
-  then
 	# run all known database configs
 	for DBCONFIG in $DBCONFIGS; do
 		execute_tests $DBCONFIG "$2"