Store storage availability in database

Storage status is saved in the database. Failed storages are rechecked every
10 minutes, while working storages are rechecked every request.

Using the files_external app will recheck all external storages when the
settings page is viewed, or whenever an external storage is saved.
This commit is contained in:
Robin McCorkell 2015-01-23 21:53:21 +00:00
parent 89d6439445
commit df19cabb44
10 changed files with 737 additions and 13 deletions

View File

@ -494,9 +494,17 @@ class OC_Mount_Config {
if (class_exists($class)) {
try {
$storage = new $class($options);
if ($storage->test($isPersonal)) {
try {
$result = $storage->test($isPersonal);
$storage->setAvailability($result);
if ($result) {
return self::STATUS_SUCCESS;
}
} catch (\Exception $e) {
$storage->setAvailability(false);
throw $e;
}
} catch (Exception $exception) {
\OCP\Util::logException('files_external', $exception);
}

View File

@ -102,6 +102,18 @@
<length>4</length>
</field>
<field>
<name>available</name>
<type>boolean</type>
<default>true</default>
<notnull>true</notnull>
</field>
<field>
<name>last_checked</name>
<type>integer</type>
</field>
<index>
<name>storages_id_index</name>
<unique>true</unique>

View File

@ -43,9 +43,10 @@ class Storage {
/**
* @param \OC\Files\Storage\Storage|string $storage
* @param bool $isAvailable
* @throws \RuntimeException
*/
public function __construct($storage) {
public function __construct($storage, $isAvailable = true) {
if ($storage instanceof \OC\Files\Storage\Storage) {
$this->storageId = $storage->getId();
} else {
@ -53,17 +54,14 @@ class Storage {
}
$this->storageId = self::adjustStorageId($this->storageId);
$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($this->storageId));
if ($row = $result->fetchRow()) {
if ($row = self::getStorageById($this->storageId)) {
$this->numericId = $row['numeric_id'];
} else {
$connection = \OC_DB::getConnection();
if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId])) {
if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $isAvailable])) {
$this->numericId = \OC_DB::insertid('*PREFIX*storages');
} else {
$result = \OC_DB::executeAudited($sql, array($this->storageId));
if ($row = $result->fetchRow()) {
if ($row = self::getStorageById($this->storageId)) {
$this->numericId = $row['numeric_id'];
} else {
throw new \RuntimeException('Storage could neither be inserted nor be selected from the database');
@ -72,6 +70,16 @@ class Storage {
}
}
/**
* @param string $storageId
* @return array|null
*/
public static function getStorageById($storageId) {
$sql = 'SELECT * FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($storageId));
return $result->fetchRow();
}
/**
* Adjusts the storage id to use md5 if too long
* @param string $storageId storage id
@ -120,15 +128,35 @@ class Storage {
public static function getNumericStorageId($storageId) {
$storageId = self::adjustStorageId($storageId);
$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
$result = \OC_DB::executeAudited($sql, array($storageId));
if ($row = $result->fetchRow()) {
if ($row = self::getStorageById($storageId)) {
return $row['numeric_id'];
} else {
return null;
}
}
/**
* @return array|null [ available, last_checked ]
*/
public function getAvailability() {
if ($row = self::getStorageById($this->storageId)) {
return [
'available' => $row['available'],
'last_checked' => $row['last_checked']
];
} else {
return null;
}
}
/**
* @param bool $isAvailable
*/
public function setAvailability($isAvailable) {
$sql = 'UPDATE `*PREFIX*storages` SET `available` = ?, `last_checked` = ? WHERE `id` = ?';
\OC_DB::executeAudited($sql, array($isAvailable, time(), $this->storageId));
}
/**
* Check if a string storage id is known
*

View File

@ -404,6 +404,11 @@ abstract class Common implements Storage {
return implode('/', $output);
}
/**
* Test a storage for availability
*
* @return bool
*/
public function test() {
if ($this->stat('')) {
return true;
@ -650,4 +655,18 @@ abstract class Common implements Storage {
public function changeLock($path, $type, ILockingProvider $provider) {
$provider->changeLock('files/' . md5($this->getId() . '::' . trim($path, '/')), $type);
}
/**
* @return array [ available, last_checked ]
*/
public function getAvailability() {
return $this->getStorageCache()->getAvailability();
}
/**
* @param bool $isAvailable
*/
public function setAvailability($isAvailable) {
$this->getStorageCache()->setAvailability($isAvailable);
}
}

View File

@ -0,0 +1,462 @@
<?php
/**
* @author Robin McCorkell <rmccorkell@karoshi.org.uk>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Files\Storage\Wrapper;
/**
* Availability checker for storages
*
* Throws a StorageNotAvailableException for storages with known failures
*/
class Availability extends Wrapper {
const RECHECK_TTL_SEC = 600; // 10 minutes
/**
* @return bool
*/
private function updateAvailability() {
try {
$result = $this->test();
} catch (\Exception $e) {
$result = false;
}
$this->setAvailability($result);
return $result;
}
/**
* @return bool
*/
private function isAvailable() {
$availability = $this->getAvailability();
if (!$availability['available']) {
// trigger a recheck if TTL reached
if ((time() - $availability['last_checked']) > self::RECHECK_TTL_SEC) {
return $this->updateAvailability();
}
}
return $availability['available'];
}
/**
* @throws \OCP\Files\StorageNotAvailableException
*/
private function checkAvailability() {
if (!$this->isAvailable()) {
throw new \OCP\Files\StorageNotAvailableException();
}
}
/** {@inheritdoc} */
public function mkdir($path) {
$this->checkAvailability();
try {
return parent::mkdir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function rmdir($path) {
$this->checkAvailability();
try {
return parent::rmdir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function opendir($path) {
$this->checkAvailability();
try {
return parent::opendir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function is_dir($path) {
$this->checkAvailability();
try {
return parent::is_dir($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function is_file($path) {
$this->checkAvailability();
try {
return parent::is_file($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function stat($path) {
$this->checkAvailability();
try {
return parent::stat($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function filetype($path) {
$this->checkAvailability();
try {
return parent::filetype($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function filesize($path) {
$this->checkAvailability();
try {
return parent::filesize($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isCreatable($path) {
$this->checkAvailability();
try {
return parent::isCreatable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isReadable($path) {
$this->checkAvailability();
try {
return parent::isReadable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isUpdatable($path) {
$this->checkAvailability();
try {
return parent::isUpdatable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isDeletable($path) {
$this->checkAvailability();
try {
return parent::isDeletable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function isSharable($path) {
$this->checkAvailability();
try {
return parent::isSharable($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getPermissions($path) {
$this->checkAvailability();
try {
return parent::getPermissions($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function file_exists($path) {
$this->checkAvailability();
try {
return parent::file_exists($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function filemtime($path) {
$this->checkAvailability();
try {
return parent::filemtime($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function file_get_contents($path) {
$this->checkAvailability();
try {
return parent::file_get_contents($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function file_put_contents($path, $data) {
$this->checkAvailability();
try {
return parent::file_put_contents($path, $data);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function unlink($path) {
$this->checkAvailability();
try {
return parent::unlink($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function rename($path1, $path2) {
$this->checkAvailability();
try {
return parent::rename($path1, $path2);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function copy($path1, $path2) {
$this->checkAvailability();
try {
return parent::copy($path1, $path2);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function fopen($path, $mode) {
$this->checkAvailability();
try {
return parent::fopen($path, $mode);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getMimeType($path) {
$this->checkAvailability();
try {
return parent::getMimeType($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function hash($type, $path, $raw = false) {
$this->checkAvailability();
try {
return parent::hash($type, $path, $raw);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function free_space($path) {
$this->checkAvailability();
try {
return parent::free_space($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function search($query) {
$this->checkAvailability();
try {
return parent::search($query);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function touch($path, $mtime = null) {
$this->checkAvailability();
try {
return parent::touch($path, $mtime);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getLocalFile($path) {
$this->checkAvailability();
try {
return parent::getLocalFile($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getLocalFolder($path) {
$this->checkAvailability();
try {
return parent::getLocalFolder($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function hasUpdated($path, $time) {
$this->checkAvailability();
try {
return parent::hasUpdated($path, $time);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getOwner($path) {
$this->checkAvailability();
try {
return parent::getOwner($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getETag($path) {
$this->checkAvailability();
try {
return parent::getETag($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getDirectDownload($path) {
$this->checkAvailability();
try {
return parent::getDirectDownload($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function copyFromStorage(\OCP\Files\Storage $sourceStorage, $sourceInternalPath, $targetInternalPath) {
$this->checkAvailability();
try {
return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function moveFromStorage(\OCP\Files\Storage $sourceStorage, $sourceInternalPath, $targetInternalPath) {
$this->checkAvailability();
try {
return parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
/** {@inheritdoc} */
public function getMetaData($path) {
$this->checkAvailability();
try {
return parent::getMetaData($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
$this->setAvailability(false);
throw $e;
}
}
}

View File

@ -497,6 +497,24 @@ class Wrapper implements \OC\Files\Storage\Storage {
return $this->storage->getDirectDownload($path);
}
/**
* Get availability of the storage
*
* @return array [ available, last_checked ]
*/
public function getAvailability() {
return $this->storage->getAvailability();
}
/**
* Set availability of the storage
*
* @param bool $isAvailable
*/
public function setAvailability($isAvailable) {
$this->storage->setAvailability($isAvailable);
}
/**
* @param string $path the path of the target folder
* @param string $fileName the name of the file itself

View File

@ -143,6 +143,14 @@ class OC_Util {
return $storage;
});
// install storage availability wrapper, before most other wrappers
\OC\Files\Filesystem::addStorageWrapper('oc_availability', function ($mountPoint, $storage) {
if (!$storage->isLocal()) {
return new \OC\Files\Storage\Wrapper\Availability(['storage' => $storage]);
}
return $storage;
});
\OC\Files\Filesystem::addStorageWrapper('oc_quota', function ($mountPoint, $storage) {
// set up quota for home storages, even for other users
// which can happen when using sharing

View File

@ -439,4 +439,24 @@ interface Storage {
* @since 8.1.0
*/
public function changeLock($path, $type, ILockingProvider $provider);
/**
* Test a storage for availability
*
* @since 8.2.0
* @return bool
*/
public function test();
/**
* @since 8.2.0
* @return array [ available, last_checked ]
*/
public function getAvailability();
/**
* @since 8.2.0
* @param bool $isAvailable
*/
public function setAvailability($isAvailable);
}

View File

@ -0,0 +1,149 @@
<?php
/**
* @author Robin McCorkell <rmccorkell@karoshi.org.uk>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace Test\Files\Storage\Wrapper;
class Availability extends \Test\TestCase {
protected function getWrapperInstance() {
$storage = $this->getMockBuilder('\OC\Files\Storage\Temporary')
->disableOriginalConstructor()
->getMock();
$wrapper = new \OC\Files\Storage\Wrapper\Availability(['storage' => $storage]);
return [$storage, $wrapper];
}
/**
* Storage is available
*/
public function testAvailable() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir');
$wrapper->mkdir('foobar');
}
/**
* Storage marked unavailable, TTL not expired
*
* @expectedException \OCP\Files\StorageNotAvailableException
*/
public function testUnavailable() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => false, 'last_checked' => time()]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->never())
->method('mkdir');
$wrapper->mkdir('foobar');
}
/**
* Storage marked unavailable, TTL expired
*/
public function testUnavailableRecheck() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => false, 'last_checked' => 0]);
$storage->expects($this->once())
->method('test')
->willReturn(true);
$storage->expects($this->once())
->method('setAvailability')
->with($this->equalTo(true));
$storage->expects($this->once())
->method('mkdir');
$wrapper->mkdir('foobar');
}
/**
* Storage marked available, but throws StorageNotAvailableException
*
* @expectedException \OCP\Files\StorageNotAvailableException
*/
public function testAvailableThrowStorageNotAvailable() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir')
->will($this->throwException(new \OCP\Files\StorageNotAvailableException()));
$storage->expects($this->once())
->method('setAvailability')
->with($this->equalTo(false));
$wrapper->mkdir('foobar');
}
/**
* Storage available, but call fails
* Method failure does not indicate storage unavailability
*/
public function testAvailableFailure() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir')
->willReturn(false);
$storage->expects($this->never())
->method('setAvailability');
$wrapper->mkdir('foobar');
}
/**
* Storage available, but throws exception
* Standard exception does not indicate storage unavailability
*
* @expectedException \Exception
*/
public function testAvailableThrow() {
list($storage, $wrapper) = $this->getWrapperInstance();
$storage->expects($this->once())
->method('getAvailability')
->willReturn(['available' => true, 'last_checked' => 0]);
$storage->expects($this->never())
->method('test');
$storage->expects($this->once())
->method('mkdir')
->will($this->throwException(new \Exception()));
$storage->expects($this->never())
->method('setAvailability');
$wrapper->mkdir('foobar');
}
}

View File

@ -22,7 +22,7 @@
// We only can count up. The 4. digit is only for the internal patchlevel to trigger DB upgrades
// between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel
// when updating major/minor version number.
$OC_Version=array(8, 2, 0, 2);
$OC_Version=array(8, 2, 0, 3);
// The human readable string
$OC_VersionString='8.2 pre alpha';