mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

drop file cache migration OC5 -> OC6

This commit is contained in:
Thomas Müller 2014-03-18 17:14:38 +01:00
parent f6c3667efa
commit 6b9ae27b90
9 changed files with 63 additions and 585 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
apps/files/ajax/upgrade.php
View File

@ -1,44 +0,0 @@
<?php
set_time_limit(0); //scanning can take ages
session_write_close();
$user = OC_User::getUser();
$eventSource = new OC_EventSource();
$listener = new UpgradeListener($eventSource);
$legacy = new \OC\Files\Cache\Legacy($user);
if ($legacy->hasItems()) {
OC_Hook::connect('\OC\Files\Cache\Upgrade', 'migrate_path', $listener, 'upgradePath');
OC_DB::beginTransaction();
$upgrade = new \OC\Files\Cache\Upgrade($legacy);
$count = $legacy->getCount();
$eventSource->send('total', $count);
$upgrade->upgradePath('/' . $user . '/files');
OC_DB::commit();
}
\OC\Files\Cache\Upgrade::upgradeDone($user);
$eventSource->send('done', true);
$eventSource->close();
class UpgradeListener {
/**
* @var OC_EventSource $eventSource
*/
private $eventSource;
private $count = 0;
private $lastSend = 0;
public function __construct($eventSource) {
$this->eventSource = $eventSource;
}
public function upgradePath($path) {
$this->count++;
if ($this->count > ($this->lastSend + 5)) {
$this->lastSend = $this->count;
$this->eventSource->send('count', $this->count);
}
}
}

137
apps/files/index.php
View File

@ -62,22 +62,17 @@ if ($isIE8 && isset($_GET['dir'])){
$ajaxLoad = false;
$files = array();
$user = OC_User::getUser();
if (\OC\Files\Cache\Upgrade::needUpgrade($user)) { //dont load anything if we need to upgrade the cache
$needUpgrade = true;
} else {
if ($isIE8){
// after the redirect above, the URL will have a format
// like "files#?dir=path" which means that no path was given
// (dir is not set). In that specific case, we don't return any
// files because the client will take care of switching the dir
// to the one from the hash, then ajax-load the initial file list
$files = array();
$ajaxLoad = true;
}
else{
$files = \OCA\Files\Helper::getFiles($dir);
}
$needUpgrade = false;
if ($isIE8){
// after the redirect above, the URL will have a format
// like "files#?dir=path" which means that no path was given
// (dir is not set). In that specific case, we don't return any
// files because the client will take care of switching the dir
// to the one from the hash, then ajax-load the initial file list
$files = array();
$ajaxLoad = true;
}
else{
$files = \OCA\Files\Helper::getFiles($dir);
}
$config = \OC::$server->getConfig();
@ -97,62 +92,56 @@ $breadcrumbNav->assign('baseURL', OCP\Util::linkTo('files', 'index.php') . '?dir
$permissions = $dirInfo->getPermissions();
if ($needUpgrade) {
OCP\Util::addscript('files', 'upgrade');
$tmpl = new OCP\Template('files', 'upgrade', 'user');
$tmpl->printPage();
} else {
// information about storage capacities
$storageInfo=OC_Helper::getStorageInfo($dir);
$freeSpace=$storageInfo['free'];
$uploadLimit=OCP\Util::uploadLimit();
$maxUploadFilesize=OCP\Util::maxUploadFilesize($dir);
$publicUploadEnabled = $config->getAppValue('core', 'shareapi_allow_public_upload', 'yes');
// if the encryption app is disabled, than everything is fine (INIT_SUCCESSFUL status code)
$encryptionInitStatus = 2;
if (OC_App::isEnabled('files_encryption')) {
$session = new \OCA\Encryption\Session(new \OC\Files\View('/'));
$encryptionInitStatus = $session->getInitialized();
}
$trashEnabled = \OCP\App::isEnabled('files_trashbin');
$trashEmpty = true;
if ($trashEnabled) {
$trashEmpty = \OCA\Files_Trashbin\Trashbin::isEmpty($user);
}
$isCreatable = \OC\Files\Filesystem::isCreatable($dir . '/');
$fileHeader = (!isset($files) or count($files) > 0);
$emptyContent = ($isCreatable and !$fileHeader) or $ajaxLoad;
OCP\Util::addscript('files', 'fileactions');
OCP\Util::addscript('files', 'files');
OCP\Util::addscript('files', 'keyboardshortcuts');
$tmpl = new OCP\Template('files', 'index', 'user');
$tmpl->assign('fileList', $list->fetchPage());
$tmpl->assign('breadcrumb', $breadcrumbNav->fetchPage());
$tmpl->assign('dir', $dir);
$tmpl->assign('isCreatable', $isCreatable);
$tmpl->assign('permissions', $permissions);
$tmpl->assign('files', $files);
$tmpl->assign('trash', $trashEnabled);
$tmpl->assign('trashEmpty', $trashEmpty);
$tmpl->assign('uploadMaxFilesize', $maxUploadFilesize); // minimium of freeSpace and uploadLimit
$tmpl->assign('uploadMaxHumanFilesize', OCP\Util::humanFileSize($maxUploadFilesize));
$tmpl->assign('freeSpace', $freeSpace);
$tmpl->assign('uploadLimit', $uploadLimit); // PHP upload limit
$tmpl->assign('allowZipDownload', intval(OCP\Config::getSystemValue('allowZipDownload', true)));
$tmpl->assign('usedSpacePercent', (int)$storageInfo['relative']);
$tmpl->assign('isPublic', false);
$tmpl->assign('publicUploadEnabled', $publicUploadEnabled);
$tmpl->assign("encryptedFiles", \OCP\Util::encryptedFiles());
$tmpl->assign("mailNotificationEnabled", $config->getAppValue('core', 'shareapi_allow_mail_notification', 'yes'));
$tmpl->assign("allowShareWithLink", $config->getAppValue('core', 'shareapi_allow_links', 'yes'));
$tmpl->assign("encryptionInitStatus", $encryptionInitStatus);
$tmpl->assign('disableSharing', false);
$tmpl->assign('ajaxLoad', $ajaxLoad);
$tmpl->assign('emptyContent', $emptyContent);
$tmpl->assign('fileHeader', $fileHeader);
$tmpl->printPage();
// information about storage capacities
$storageInfo=OC_Helper::getStorageInfo($dir);
$freeSpace=$storageInfo['free'];
$uploadLimit=OCP\Util::uploadLimit();
$maxUploadFilesize=OCP\Util::maxUploadFilesize($dir);
$publicUploadEnabled = $config->getAppValue('core', 'shareapi_allow_public_upload', 'yes');
// if the encryption app is disabled, than everything is fine (INIT_SUCCESSFUL status code)
$encryptionInitStatus = 2;
if (OC_App::isEnabled('files_encryption')) {
$session = new \OCA\Encryption\Session(new \OC\Files\View('/'));
$encryptionInitStatus = $session->getInitialized();
}
$trashEnabled = \OCP\App::isEnabled('files_trashbin');
$trashEmpty = true;
if ($trashEnabled) {
$trashEmpty = \OCA\Files_Trashbin\Trashbin::isEmpty($user);
}
$isCreatable = \OC\Files\Filesystem::isCreatable($dir . '/');
$fileHeader = (!isset($files) or count($files) > 0);
$emptyContent = ($isCreatable and !$fileHeader) or $ajaxLoad;
OCP\Util::addscript('files', 'fileactions');
OCP\Util::addscript('files', 'files');
OCP\Util::addscript('files', 'keyboardshortcuts');
$tmpl = new OCP\Template('files', 'index', 'user');
$tmpl->assign('fileList', $list->fetchPage());
$tmpl->assign('breadcrumb', $breadcrumbNav->fetchPage());
$tmpl->assign('dir', $dir);
$tmpl->assign('isCreatable', $isCreatable);
$tmpl->assign('permissions', $permissions);
$tmpl->assign('files', $files);
$tmpl->assign('trash', $trashEnabled);
$tmpl->assign('trashEmpty', $trashEmpty);
$tmpl->assign('uploadMaxFilesize', $maxUploadFilesize); // minimium of freeSpace and uploadLimit
$tmpl->assign('uploadMaxHumanFilesize', OCP\Util::humanFileSize($maxUploadFilesize));
$tmpl->assign('freeSpace', $freeSpace);
$tmpl->assign('uploadLimit', $uploadLimit); // PHP upload limit
$tmpl->assign('allowZipDownload', intval(OCP\Config::getSystemValue('allowZipDownload', true)));
$tmpl->assign('usedSpacePercent', (int)$storageInfo['relative']);
$tmpl->assign('isPublic', false);
$tmpl->assign('publicUploadEnabled', $publicUploadEnabled);
$tmpl->assign("encryptedFiles", \OCP\Util::encryptedFiles());
$tmpl->assign("mailNotificationEnabled", $config->getAppValue('core', 'shareapi_allow_mail_notification', 'yes'));
$tmpl->assign("allowShareWithLink", $config->getAppValue('core', 'shareapi_allow_links', 'yes'));
$tmpl->assign("encryptionInitStatus", $encryptionInitStatus);
$tmpl->assign('disableSharing', false);
$tmpl->assign('ajaxLoad', $ajaxLoad);
$tmpl->assign('emptyContent', $emptyContent);
$tmpl->assign('fileHeader', $fileHeader);
$tmpl->printPage();

28
apps/files/js/upgrade.js
View File

@ -1,28 +0,0 @@
/*
* Copyright (c) 2014
*
* This file is licensed under the Affero General Public License version 3
* or later.
*
* See the COPYING-README file.
*
*/
/* global OC */
$(document).ready(function () {
var eventSource, total, bar = $('#progressbar');
console.log('start');
bar.progressbar({value: 0});
eventSource = new OC.EventSource(OC.filePath('files', 'ajax', 'upgrade.php'));
eventSource.listen('total', function (count) {
total = count;
console.log(count + ' files needed to be migrated');
});
eventSource.listen('count', function (count) {
bar.progressbar({value: (count / total) * 100});
console.log(count);
});
eventSource.listen('done', function () {
document.location.reload();
});
});

4
apps/files/templates/upgrade.php
View File

@ -1,4 +0,0 @@
<div id="upgrade">
<?php p($l->t('Upgrading filesystem cache...'));?>
<div id="progressbar" />
</div>

9
core/ajax/update.php
View File

@ -16,15 +16,6 @@ if (OC::checkUpgrade(false)) {
$updater->listen('\OC\Updater', 'dbUpgrade', function () use ($eventSource, $l) {
$eventSource->send('success', (string)$l->t('Updated database'));
});
$updater->listen('\OC\Updater', 'filecacheStart', function () use ($eventSource, $l) {
$eventSource->send('success', (string)$l->t('Updating filecache, this may take really long...'));
});
$updater->listen('\OC\Updater', 'filecacheDone', function () use ($eventSource, $l) {
$eventSource->send('success', (string)$l->t('Updated filecache'));
});
$updater->listen('\OC\Updater', 'filecacheProgress', function ($out) use ($eventSource, $l) {
$eventSource->send('success', (string)$l->t('... %d%% done ...', array('percent' => $out)));
});
$updater->listen('\OC\Updater', 'failure', function ($message) use ($eventSource) {
$eventSource->send('failure', $message);
$eventSource->close();

9
core/command/upgrade.php
View File

@ -59,15 +59,6 @@ class Upgrade extends Command {
$updater->listen('\OC\Updater', 'dbUpgrade', function () use($output) {
$output->writeln('<info>Updated database</info>');
});
$updater->listen('\OC\Updater', 'filecacheStart', function () use($output) {
$output->writeln('<info>Updating filecache, this may take really long...</info>');
});
$updater->listen('\OC\Updater', 'filecacheDone', function () use($output) {
$output->writeln('<info>Updated filecache</info>');
});
$updater->listen('\OC\Updater', 'filecacheProgress', function ($out) use($output) {
$output->writeln('... ' . $out . '% done ...');
});
$updater->listen('\OC\Updater', 'failure', function ($message) use($output) {
$output->writeln($message);

139
lib/private/files/cache/legacy.php vendored
View File

@ -1,139 +0,0 @@
<?php
/**
* Copyright (c) 2012 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\Files\Cache;
/**
* Provide read only support for the old filecache
*/
class Legacy {
private $user;
private $cacheHasItems = null;
/**
* @param string $user
*/
public function __construct($user) {
$this->user = $user;
}
/**
* get the numbers of items in the legacy cache
*
* @return int
*/
function getCount() {
$sql = 'SELECT COUNT(`id`) AS `count` FROM `*PREFIX*fscache` WHERE `user` = ?';
$result = \OC_DB::executeAudited($sql, array($this->user));
if ($row = $result->fetchRow()) {
return $row['count'];
} else {
return 0;
}
}
/**
* check if a legacy cache is present and holds items
*
* @return bool
*/
function hasItems() {
if (!is_null($this->cacheHasItems)) {
return $this->cacheHasItems;
}
try {
$query = \OC_DB::prepare('SELECT `id` FROM `*PREFIX*fscache` WHERE `user` = ?',1);
} catch (\Exception $e) {
$this->cacheHasItems = false;
return false;
}
try {
$result = $query->execute(array($this->user));
} catch (\Exception $e) {
$this->cacheHasItems = false;
return false;
}
if ($result === false || property_exists($result, 'error_message_prefix')) {
$this->cacheHasItems = false;
return false;
}
$this->cacheHasItems = (bool)$result->fetchRow();
return $this->cacheHasItems;
}
/**
* get an item from the legacy cache
*
* @param string $path
* @return array
*/
function get($path) {
if (is_numeric($path)) {
$sql = 'SELECT * FROM `*PREFIX*fscache` WHERE `id` = ?';
} else {
$sql = 'SELECT * FROM `*PREFIX*fscache` WHERE `path` = ?';
}
$result = \OC_DB::executeAudited($sql, array($path));
$data = $result->fetchRow();
$data['etag'] = $this->getEtag($data['path'], $data['user']);
return $data;
}
/**
* Get the ETag for the given path
*
* @param type $path
* @return string
*/
function getEtag($path, $user = null) {
static $query = null;
$pathDetails = explode('/', $path, 4);
if((!$user) && !isset($pathDetails[1])) {
//no user!? Too odd, return empty string.
return '';
} else if(!$user) {
//guess user from path, if no user passed.
$user = $pathDetails[1];
}
if(!isset($pathDetails[3]) || is_null($pathDetails[3])) {
$relativePath = '';
} else {
$relativePath = $pathDetails[3];
}
if(is_null($query)){
$query = \OC_DB::prepare('SELECT `propertyvalue` FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ? AND `propertyname` = \'{DAV:}getetag\'');
}
$result = \OC_DB::executeAudited($query,array($user, '/' . $relativePath));
if ($row = $result->fetchRow()) {
return trim($row['propertyvalue'], '"');
} else {
return '';
}
}
/**
* get all child items of an item from the legacy cache
*
* @param int $id
* @return array
*/
function getChildren($id) {
$result = \OC_DB::executeAudited('SELECT * FROM `*PREFIX*fscache` WHERE `parent` = ?', array($id));
$data = $result->fetchAll();
foreach ($data as $i => $item) {
$data[$i]['etag'] = $this->getEtag($item['path'], $item['user']);
}
return $data;
}
}

235
lib/private/files/cache/upgrade.php vendored
View File

@ -1,235 +0,0 @@
<?php
/**
* Copyright (c) 2012 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\Files\Cache;
class Upgrade {
/**
* @var Legacy $legacy
*/
private $legacy;
private $numericIds = array();
private $mimeTypeIds = array();
/**
* @param Legacy $legacy
*/
public function __construct($legacy) {
$this->legacy = $legacy;
}
/**
* Preform a upgrade a path and it's childs
*
* @param string $path
* @param bool $mode
*/
function upgradePath($path, $mode = Scanner::SCAN_RECURSIVE) {
if (!$this->legacy->hasItems()) {
return;
}
\OC_Hook::emit('\OC\Files\Cache\Upgrade', 'migrate_path', $path);
if ($row = $this->legacy->get($path)) {
$data = $this->getNewData($row);
if ($data) {
$this->insert($data);
$this->upgradeChilds($data['id'], $mode);
}
}
}
/**
* upgrade all child elements of an item
*
* @param int $id
* @param bool $mode
*/
function upgradeChilds($id, $mode = Scanner::SCAN_RECURSIVE) {
$children = $this->legacy->getChildren($id);
foreach ($children as $child) {
$childData = $this->getNewData($child);
\OC_Hook::emit('\OC\Files\Cache\Upgrade', 'migrate_path', $child['path']);
if ($childData) {
$this->insert($childData);
if ($mode == Scanner::SCAN_RECURSIVE) {
$this->upgradeChilds($child['id']);
}
}
}
}
/**
* insert data into the new cache
*
* @param array $data the data for the new cache
*/
function insert($data) {
static $insertQuery = null;
if(is_null($insertQuery)) {
$insertQuery = \OC_DB::prepare('INSERT INTO `*PREFIX*filecache`
( `fileid`, `storage`, `path`, `path_hash`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` )
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
}
if (!$this->inCache($data['storage'], $data['path_hash'], $data['id'])) {
\OC_DB::executeAudited($insertQuery, array($data['id'], $data['storage'],
$data['path'], $data['path_hash'], $data['parent'], $data['name'],
$data['mimetype'], $data['mimepart'], $data['size'], $data['mtime'], $data['encrypted'], $data['etag']));
}
}
/**
* check if an item is already in the new cache
*
* @param string $storage
* @param string $pathHash
* @param string $id
* @return bool
*/
function inCache($storage, $pathHash, $id) {
static $query = null;
if(is_null($query)) {
$query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE (`storage` = ? AND `path_hash` = ?) OR `fileid` = ?');
}
$result = \OC_DB::executeAudited($query, array($storage, $pathHash, $id));
return (bool)$result->fetchRow();
}
/**
* get the new data array from the old one
*
* @param array $data the data from the old cache
* Example data array
* Array
* (
* [id] => 418
* [path] => /tina/files/picture.jpg //relative to datadir
* [path_hash] => 66d4547e372888deed80b24fec9b192b
* [parent] => 234
* [name] => picture.jpg
* [user] => tina
* [size] => 1265283
* [ctime] => 1363909709
* [mtime] => 1363909709
* [mimetype] => image/jpeg
* [mimepart] => image
* [encrypted] => 0
* [versioned] => 0
* [writable] => 1
* )
*
* @return array
*/
function getNewData($data) {
//Make sure there is a path, otherwise we can do nothing.
if(!isset($data['path'])) {
return false;
}
$newData = $data;
/**
* @var \OC\Files\Storage\Storage $storage
* @var string $internalPath;
*/
list($storage, $internalPath) = \OC\Files\Filesystem::resolvePath($data['path']);
if ($storage) {
$newData['etag'] = $data['etag'];
$newData['path_hash'] = md5($internalPath);
$newData['path'] = $internalPath;
$newData['storage'] = $this->getNumericId($storage);
$newData['parent'] = ($internalPath === '') ? -1 : $data['parent'];
$newData['permissions'] = ($data['writable']) ? \OCP\PERMISSION_ALL : \OCP\PERMISSION_READ;
$newData['storage_object'] = $storage;
$newData['mimetype'] = $this->getMimetypeId($newData['mimetype'], $storage);
$newData['mimepart'] = $this->getMimetypeId($newData['mimepart'], $storage);
return $newData;
} else {
\OC_Log::write('core', 'Unable to migrate data from old cache for '.$data['path'].' because the storage was not found', \OC_Log::ERROR);
return false;
}
}
/**
* get the numeric storage id
*
* @param \OC\Files\Storage\Storage $storage
* @return int
*/
function getNumericId($storage) {
$storageId = $storage->getId();
if (!isset($this->numericIds[$storageId])) {
$cache = $storage->getCache();
$this->numericIds[$storageId] = $cache->getNumericStorageId();
}
return $this->numericIds[$storageId];
}
/**
* get the numeric id for a mimetype
*
* @param string $mimetype
* @param \OC\Files\Storage\Storage $storage
* @return int
*/
function getMimetypeId($mimetype, $storage) {
if (!isset($this->mimeTypeIds[$mimetype])) {
$cache = new Cache($storage);
$this->mimeTypeIds[$mimetype] = $cache->getMimetypeId($mimetype);
}
return $this->mimeTypeIds[$mimetype];
}
/**
* check if a cache upgrade is required for $user
*
* @param string $user
* @return bool
*/
static function needUpgrade($user) {
$cacheVersion = (int)\OCP\Config::getUserValue($user, 'files', 'cache_version', 4);
if ($cacheVersion < 5) {
$legacy = new \OC\Files\Cache\Legacy($user);
if ($legacy->hasItems()) {
return true;
}
self::upgradeDone($user);
}
return false;
}
/**
* mark the filecache as upgrade
*
* @param string $user
*/
static function upgradeDone($user) {
\OCP\Config::setUserValue($user, 'files', 'cache_version', 5);
}
/**
* Does a "silent" upgrade, i.e. without an Event-Source as triggered
* on User-Login via Ajax. This method is called within the regular
* ownCloud upgrade.
*
* @param string $user a User ID
*/
public static function doSilentUpgrade($user) {
if(!self::needUpgrade($user)) {
return;
}
$legacy = new \OC\Files\Cache\Legacy($user);
if ($legacy->hasItems()) {
\OC_DB::beginTransaction();
$upgrade = new \OC\Files\Cache\Upgrade($legacy);
$upgrade->upgradePath('/' . $user . '/files');
\OC_DB::commit();
}
\OC\Files\Cache\Upgrade::upgradeDone($user);
}
}

43
lib/private/updater.php
View File

@ -16,9 +16,6 @@ use OC\Hooks\BasicEmitter;
* - maintenanceStart()
* - maintenanceEnd()
* - dbUpgrade()
* - filecacheStart()
* - filecacheProgress(int $percentage)
* - filecacheDone()
* - failure(string $message)
*/
class Updater extends BasicEmitter {
@ -122,9 +119,6 @@ class Updater extends BasicEmitter {
\OC_DB::updateDbFromStructure(\OC::$SERVERROOT . '/db_structure.xml');
$this->emit('\OC\Updater', 'dbUpgrade');
// do a file cache upgrade for users with files
// this can take loooooooooooooooooooooooong
$this->upgradeFileCache();
} catch (\Exception $exception) {
$this->emit('\OC\Updater', 'failure', array($exception->getMessage()));
}
@ -142,42 +136,5 @@ class Updater extends BasicEmitter {
$this->emit('\OC\Updater', 'maintenanceEnd');
}
private function upgradeFileCache() {
try {
$query = \OC_DB::prepare('
SELECT DISTINCT `user`
FROM `*PREFIX*fscache`
');
$result = $query->execute();
} catch (\Exception $e) {
return;
}
$users = $result->fetchAll();
if (count($users) == 0) {
return;
}
$step = 100 / count($users);
$percentCompleted = 0;
$lastPercentCompletedOutput = 0;
$startInfoShown = false;
foreach ($users as $userRow) {
$user = $userRow['user'];
\OC\Files\Filesystem::initMountPoints($user);
\OC\Files\Cache\Upgrade::doSilentUpgrade($user);
if (!$startInfoShown) {
//We show it only now, because otherwise Info about upgraded apps
//will appear between this and progress info
$this->emit('\OC\Updater', 'filecacheStart');
$startInfoShown = true;
}
$percentCompleted += $step;
$out = floor($percentCompleted);
if ($out != $lastPercentCompletedOutput) {
$this->emit('\OC\Updater', 'filecacheProgress', array($out));
$lastPercentCompletedOutput = $out;
}
}
$this->emit('\OC\Updater', 'filecacheDone');
}
}