From b1e778d5fd05ae7224e73dc2cdc72a94c4f8b55c Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Wed, 25 Jun 2014 19:34:39 +0200 Subject: [PATCH 1/4] Moved office mimetype update to a repair step --- apps/files/appinfo/update.php | 57 -------------- lib/private/repair.php | 7 +- lib/repair/repairmimetypes.php | 87 ++++++++++++++++++++++ tests/lib/repair/repairmimetypes.php | 107 +++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 59 deletions(-) create mode 100644 lib/repair/repairmimetypes.php create mode 100644 tests/lib/repair/repairmimetypes.php diff --git a/apps/files/appinfo/update.php b/apps/files/appinfo/update.php index a9b8ccbf82..de635e5ce6 100644 --- a/apps/files/appinfo/update.php +++ b/apps/files/appinfo/update.php @@ -6,60 +6,3 @@ if (version_compare(\OCP\Config::getSystemValue('version', '0.0.0'), '7.0.0', '< \OCP\Config::deleteSystemValue('allowZipDownload'); \OCP\Config::deleteSystemValue('maxZipInputSize'); } - -if (version_compare(\OCP\Config::getAppValue('files', 'installed_version'), '1.1.9', '<')) { - - // update wrong mimetypes - $wrongMimetypes = array( - 'application/mspowerpoint' => 'application/vnd.ms-powerpoint', - 'application/msexcel' => 'application/vnd.ms-excel', - ); - - $stmt = OC_DB::prepare(' - UPDATE `*PREFIX*mimetypes` - SET `mimetype` = ? - WHERE `mimetype` = ? - '); - - foreach ($wrongMimetypes as $wrong => $correct) { - OC_DB::executeAudited($stmt, array($wrong, $correct)); - } - - $updatedMimetypes = array( - 'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'xlsx' => 'application/vnd.ms-excel', - 'pptx' => 'application/vnd.ms-powerpoint', - ); - - // separate doc from docx etc - foreach ($updatedMimetypes as $extension => $mimetype ) { - $result = OC_DB::executeAudited(' - SELECT count(`mimetype`) - FROM `*PREFIX*mimetypes` - WHERE `mimetype` = ? - ', array($mimetype) - ); - - $exists = $result->fetchOne(); - - if ( ! $exists ) { - // insert mimetype - OC_DB::executeAudited(' - INSERT INTO `*PREFIX*mimetypes` ( `mimetype` ) - VALUES ( ? ) - ', array($mimetype) - ); - } - - // change mimetype for files with x extension - OC_DB::executeAudited(' - UPDATE `*PREFIX*filecache` - SET `mimetype` = ( - SELECT `id` - FROM `*PREFIX*mimetypes` - WHERE `mimetype` = ? - ) WHERE `name` LIKE ? - ', array($mimetype, '%.'.$extension) - ); - } -} \ No newline at end of file diff --git a/lib/private/repair.php b/lib/private/repair.php index 23d1c2b831..93547fd675 100644 --- a/lib/private/repair.php +++ b/lib/private/repair.php @@ -68,7 +68,9 @@ class Repair extends BasicEmitter { * @return array of RepairStep instances */ public static function getRepairSteps() { - return array(); + return array( + new \OC\Repair\RepairMimeTypes() + ); } /** @@ -78,6 +80,7 @@ class Repair extends BasicEmitter { * @return array of RepairStep instances */ public static function getBeforeUpgradeRepairSteps() { - return array(); + return array( + ); } } diff --git a/lib/repair/repairmimetypes.php b/lib/repair/repairmimetypes.php new file mode 100644 index 0000000000..397d18eb72 --- /dev/null +++ b/lib/repair/repairmimetypes.php @@ -0,0 +1,87 @@ + + * Copyright (c) 2014 Jörn Dreyer jfd@owncloud.com + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\Repair; + +use OC\Hooks\BasicEmitter; + +class RepairMimeTypes extends BasicEmitter implements \OC\RepairStep { + + public function getName() { + return 'Repair mime types'; + } + + private function fixOfficeMimeTypes() { + // update wrong mimetypes + $wrongMimetypes = array( + 'application/mspowerpoint' => 'application/vnd.ms-powerpoint', + 'application/msexcel' => 'application/vnd.ms-excel', + ); + + $stmt = \OC_DB::prepare(' + UPDATE `*PREFIX*mimetypes` + SET `mimetype` = ? + WHERE `mimetype` = ? + '); + + foreach ($wrongMimetypes as $wrong => $correct) { + \OC_DB::executeAudited($stmt, array($wrong, $correct)); + } + + $updatedMimetypes = array( + 'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'xlsx' => 'application/vnd.ms-excel', + 'pptx' => 'application/vnd.ms-powerpoint', + ); + + // separate doc from docx etc + foreach ($updatedMimetypes as $extension => $mimetype ) { + $result = \OC_DB::executeAudited(' + SELECT count(`mimetype`) + FROM `*PREFIX*mimetypes` + WHERE `mimetype` = ? + ', array($mimetype) + ); + + $exists = $result->fetchOne(); + + if ( ! $exists ) { + // insert mimetype + \OC_DB::executeAudited(' + INSERT INTO `*PREFIX*mimetypes` ( `mimetype` ) + VALUES ( ? ) + ', array($mimetype) + ); + } + + // change mimetype for files with x extension + \OC_DB::executeAudited(' + UPDATE `*PREFIX*filecache` + SET `mimetype` = ( + SELECT `id` + FROM `*PREFIX*mimetypes` + WHERE `mimetype` = ? + ) WHERE `name` LIKE ? + ', array($mimetype, '%.'.$extension) + ); + } + return true; + } + + /** + * Fix mime types + */ + public function run() { + // TODO: check precondition to avoid running the fix every time + if ($this->fixOfficeMimeTypes()) { + $this->emit('\OC\Repair', 'info', array('Fixed office mime types')); + } + } +} + diff --git a/tests/lib/repair/repairmimetypes.php b/tests/lib/repair/repairmimetypes.php new file mode 100644 index 0000000000..42af116b72 --- /dev/null +++ b/tests/lib/repair/repairmimetypes.php @@ -0,0 +1,107 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +/** + * Tests for the converting of legacy storages to home storages. + * + * @see \OC\Repair\RepairMimeTypes + */ +class TestRepairMimeTypes extends PHPUnit_Framework_TestCase { + + /** @var \OC\RepairStep */ + private $repair; + + private $storage; + + public function setUp() { + $this->storage = new \OC\Files\Storage\Temporary(array()); + + $this->repair = new \OC\Repair\RepairMimeTypes(); + } + + public function tearDown() { + $this->storage->getCache()->clear(); + $sql = 'DELETE FROM `*PREFIX*storages` WHERE id = ?'; + \OC_DB::executeAudited($sql, array($this->storage->getId())); + $this->clearMimeTypes(); + + DummyFileCache::clearCachedMimeTypes(); + } + + private function clearMimeTypes() { + $sql = 'DELETE FROM `*PREFIX*mimetypes`'; + \OC_DB::executeAudited($sql); + } + + private function addEntries($entries) { + // create files for the different extensions, this + // will also automatically create the corresponding mime types + foreach ($entries as $entry) { + $this->storage->getCache()->put( + $entry[0], + array( + 'size' => 0, + 'mtime' => 0, + 'mimetype' => $entry[1] + ) + ); + } + + } + + private function checkEntries($entries) { + foreach ($entries as $entry) { + $data = $this->storage->getCache()->get($entry[0]); + $this->assertEquals($entry[1], $data['mimetype']); + } + } + + /** + * Test renaming and splitting old office mime types + */ + public function testRenameOfficeMimeTypes() { + $this->addEntries( + array( + array('test.doc', 'application/msword'), + array('test.docx', 'application/msword'), + array('test.xls', 'application/msexcel'), + array('test.xlsx', 'application/msexcel'), + array('test.ppt', 'application/mspowerpoint'), + array('test.pptx', 'application/mspowerpoint'), + ) + ); + + $this->repair->run(); + + // force mimetype reload + $this->storage->getCache()->loadMimeTypes(); + + $this->checkEntries( + array( + array('test.doc', 'application/msword'), + array('test.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + array('test.xls', 'application/msexcel'), + array('test.xlsx', 'application/vnd.ms-excel'), + array('test.ppt', 'application/mspowerpoint'), + array('test.pptx', 'application/vnd.ms-powerpoint'), + ) + ); + } +} + +/** + * Dummy class to access protected members + */ +class DummyFileCache extends \OC\Files\Cache\Cache { + + public static function clearCachedMimeTypes() { + self::$mimetypeIds = array(); + self::$mimetypes = array(); + } +} + From b749292c18367a653d11d9d93e3320cf1dc9d330 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Wed, 25 Jun 2014 19:35:05 +0200 Subject: [PATCH 2/4] Added test case for when target mime types already exist --- tests/lib/repair/repairmimetypes.php | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/lib/repair/repairmimetypes.php b/tests/lib/repair/repairmimetypes.php index 42af116b72..91600502da 100644 --- a/tests/lib/repair/repairmimetypes.php +++ b/tests/lib/repair/repairmimetypes.php @@ -92,6 +92,52 @@ class TestRepairMimeTypes extends PHPUnit_Framework_TestCase { ) ); } + + /** + * Test renaming and splitting old office mime types when + * new ones already exist + */ + public function testRenameOfficeMimeTypesWhenExist() { + $this->addEntries( + array( + array('test.doc', 'application/msword'), + array('test.docx', 'application/msword'), + array('test.xls', 'application/msexcel'), + array('test.xlsx', 'application/msexcel'), + array('test.ppt', 'application/mspowerpoint'), + array('test.pptx', 'application/mspowerpoint'), + // make it so that the new mimetypes already exist + array('bogus.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + array('bogus.xlsx', 'application/vnd.ms-excel'), + array('bogus.pptx', 'application/vnd.ms-powerpoint'), + array('bogus2.docx', 'application/wrong'), + array('bogus2.xlsx', 'application/wrong'), + array('bogus2.pptx', 'application/wrong'), + ) + ); + + $this->repair->run(); + + // force mimetype reload + $this->storage->getCache()->loadMimeTypes(); + + $this->checkEntries( + array( + array('test.doc', 'application/msword'), + array('test.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + array('test.xls', 'application/msexcel'), + array('test.xlsx', 'application/vnd.ms-excel'), + array('test.ppt', 'application/mspowerpoint'), + array('test.pptx', 'application/vnd.ms-powerpoint'), + array('bogus.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + array('bogus.xlsx', 'application/vnd.ms-excel'), + array('bogus.pptx', 'application/vnd.ms-powerpoint'), + array('bogus2.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + array('bogus2.xlsx', 'application/vnd.ms-excel'), + array('bogus2.pptx', 'application/vnd.ms-powerpoint'), + ) + ); + } } /** From 67d0a3c15e52d865ebb5420e98b177a1bb9e42ba Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Thu, 26 Jun 2014 16:40:12 +0200 Subject: [PATCH 3/4] Fixed mime type repair step --- lib/repair/repairmimetypes.php | 96 +++++++++++++++++++--------- tests/lib/repair/repairmimetypes.php | 78 ++++++++++++++++++---- 2 files changed, 133 insertions(+), 41 deletions(-) diff --git a/lib/repair/repairmimetypes.php b/lib/repair/repairmimetypes.php index 397d18eb72..f7618c6e06 100644 --- a/lib/repair/repairmimetypes.php +++ b/lib/repair/repairmimetypes.php @@ -24,61 +24,99 @@ class RepairMimeTypes extends BasicEmitter implements \OC\RepairStep { 'application/msexcel' => 'application/vnd.ms-excel', ); - $stmt = \OC_DB::prepare(' - UPDATE `*PREFIX*mimetypes` - SET `mimetype` = ? + $existsStmt = \OC_DB::prepare(' + SELECT count(`mimetype`) + FROM `*PREFIX*mimetypes` WHERE `mimetype` = ? '); + $getIdStmt = \OC_DB::prepare(' + SELECT `id` + FROM `*PREFIX*mimetypes` + WHERE `mimetype` = ? + '); + + $insertStmt = \OC_DB::prepare(' + INSERT INTO `*PREFIX*mimetypes` ( `mimetype` ) + VALUES ( ? ) + '); + + $updateWrongStmt = \OC_DB::prepare(' + UPDATE `*PREFIX*filecache` + SET `mimetype` = ( + SELECT `id` + FROM `*PREFIX*mimetypes` + WHERE `mimetype` = ? + ) WHERE `mimetype` = ? + '); + + $deleteStmt = \OC_DB::prepare(' + DELETE FROM `*PREFIX*mimetypes` + WHERE `id` = ? + '); + foreach ($wrongMimetypes as $wrong => $correct) { - \OC_DB::executeAudited($stmt, array($wrong, $correct)); + + + // do we need to remove a wrong mimetype? + $result = \OC_DB::executeAudited($getIdStmt, array($wrong)); + $wrongId = $result->fetchOne(); + + if ($wrongId !== false) { + + // do we need to insert the correct mimetype? + $result = \OC_DB::executeAudited($existsStmt, array($correct)); + $exists = $result->fetchOne(); + + if ( ! $exists ) { + // insert mimetype + \OC_DB::executeAudited($insertStmt, array($correct)); + } + + // change wrong mimetype to correct mimetype in filecache + \OC_DB::executeAudited($updateWrongStmt, array($correct, $wrongId)); + + // delete wrong mimetype + \OC_DB::executeAudited($deleteStmt, array($wrongId)); + + } + } $updatedMimetypes = array( 'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'xlsx' => 'application/vnd.ms-excel', - 'pptx' => 'application/vnd.ms-powerpoint', + 'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation', ); + $updateByNameStmt = \OC_DB::prepare(' + UPDATE `*PREFIX*filecache` + SET `mimetype` = ( + SELECT `id` + FROM `*PREFIX*mimetypes` + WHERE `mimetype` = ? + ) WHERE `name` LIKE ? + '); + // separate doc from docx etc foreach ($updatedMimetypes as $extension => $mimetype ) { - $result = \OC_DB::executeAudited(' - SELECT count(`mimetype`) - FROM `*PREFIX*mimetypes` - WHERE `mimetype` = ? - ', array($mimetype) - ); - + $result = \OC_DB::executeAudited($existsStmt, array($mimetype)); $exists = $result->fetchOne(); if ( ! $exists ) { // insert mimetype - \OC_DB::executeAudited(' - INSERT INTO `*PREFIX*mimetypes` ( `mimetype` ) - VALUES ( ? ) - ', array($mimetype) - ); + \OC_DB::executeAudited($insertStmt, array($mimetype)); } // change mimetype for files with x extension - \OC_DB::executeAudited(' - UPDATE `*PREFIX*filecache` - SET `mimetype` = ( - SELECT `id` - FROM `*PREFIX*mimetypes` - WHERE `mimetype` = ? - ) WHERE `name` LIKE ? - ', array($mimetype, '%.'.$extension) - ); + \OC_DB::executeAudited($updateByNameStmt, array($mimetype, '%.'.$extension)); } - return true; } /** * Fix mime types */ public function run() { - // TODO: check precondition to avoid running the fix every time if ($this->fixOfficeMimeTypes()) { $this->emit('\OC\Repair', 'info', array('Fixed office mime types')); } diff --git a/tests/lib/repair/repairmimetypes.php b/tests/lib/repair/repairmimetypes.php index 91600502da..5c0a36d469 100644 --- a/tests/lib/repair/repairmimetypes.php +++ b/tests/lib/repair/repairmimetypes.php @@ -61,6 +61,20 @@ class TestRepairMimeTypes extends PHPUnit_Framework_TestCase { } } + /** + * Returns the id of a given mime type or null + * if it does not exist. + */ + private function getMimeTypeIdFromDB($mimeType) { + $sql = 'SELECT `id` FROM `*PREFIX*mimetypes` WHERE mimetype = ?'; + $results = \OC_DB::executeAudited($sql, array($mimeType)); + $result = $results->fetchOne(); + if ($result) { + return $result['id']; + } + return null; + } + /** * Test renaming and splitting old office mime types */ @@ -79,16 +93,17 @@ class TestRepairMimeTypes extends PHPUnit_Framework_TestCase { $this->repair->run(); // force mimetype reload + DummyFileCache::clearCachedMimeTypes(); $this->storage->getCache()->loadMimeTypes(); $this->checkEntries( array( array('test.doc', 'application/msword'), array('test.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), - array('test.xls', 'application/msexcel'), - array('test.xlsx', 'application/vnd.ms-excel'), - array('test.ppt', 'application/mspowerpoint'), - array('test.pptx', 'application/vnd.ms-powerpoint'), + array('test.xls', 'application/vnd.ms-excel'), + array('test.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'), + array('test.ppt', 'application/vnd.ms-powerpoint'), + array('test.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'), ) ); } @@ -119,22 +134,61 @@ class TestRepairMimeTypes extends PHPUnit_Framework_TestCase { $this->repair->run(); // force mimetype reload + DummyFileCache::clearCachedMimeTypes(); $this->storage->getCache()->loadMimeTypes(); $this->checkEntries( array( array('test.doc', 'application/msword'), array('test.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), - array('test.xls', 'application/msexcel'), - array('test.xlsx', 'application/vnd.ms-excel'), - array('test.ppt', 'application/mspowerpoint'), - array('test.pptx', 'application/vnd.ms-powerpoint'), + array('test.xls', 'application/vnd.ms-excel'), + array('test.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'), + array('test.ppt', 'application/vnd.ms-powerpoint'), + array('test.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'), array('bogus.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), - array('bogus.xlsx', 'application/vnd.ms-excel'), - array('bogus.pptx', 'application/vnd.ms-powerpoint'), + array('bogus.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'), + array('bogus.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'), array('bogus2.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), - array('bogus2.xlsx', 'application/vnd.ms-excel'), - array('bogus2.pptx', 'application/vnd.ms-powerpoint'), + array('bogus2.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'), + array('bogus2.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'), + ) + ); + + // wrong mimetypes are gone + $this->assertNull($this->getMimeTypeIdFromDB('application/msexcel')); + $this->assertNull($this->getMimeTypeIdFromDB('application/mspowerpoint')); + } + + /** + * Test that nothing happens and no error happens when all mimetypes are + * already correct and no old ones exist.. + */ + public function testDoNothingWhenOnlyNewFiles() { + $this->addEntries( + array( + array('test.doc', 'application/msword'), + array('test.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + array('test.xls', 'application/vnd.ms-excel'), + array('test.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'), + array('test.ppt', 'application/vnd.ms-powerpoint'), + array('test.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'), + ) + ); + + $this->repair->run(); + + // force mimetype reload + DummyFileCache::clearCachedMimeTypes(); + $this->storage->getCache()->loadMimeTypes(); + + $this->checkEntries( + array( + array('test.doc', 'application/msword'), + array('test.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + array('test.xls', 'application/vnd.ms-excel'), + array('test.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'), + array('test.ppt', 'application/vnd.ms-powerpoint'), + array('test.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'), ) ); } From 7ee90ddd595ab51d76bd95809dbb1bd997096e10 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Thu, 26 Jun 2014 19:24:46 +0200 Subject: [PATCH 4/4] Add missing quotes for Oracle --- tests/lib/repair/repairmimetypes.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lib/repair/repairmimetypes.php b/tests/lib/repair/repairmimetypes.php index 5c0a36d469..3ed19bd55b 100644 --- a/tests/lib/repair/repairmimetypes.php +++ b/tests/lib/repair/repairmimetypes.php @@ -26,7 +26,7 @@ class TestRepairMimeTypes extends PHPUnit_Framework_TestCase { public function tearDown() { $this->storage->getCache()->clear(); - $sql = 'DELETE FROM `*PREFIX*storages` WHERE id = ?'; + $sql = 'DELETE FROM `*PREFIX*storages` WHERE `id` = ?'; \OC_DB::executeAudited($sql, array($this->storage->getId())); $this->clearMimeTypes(); @@ -66,7 +66,7 @@ class TestRepairMimeTypes extends PHPUnit_Framework_TestCase { * if it does not exist. */ private function getMimeTypeIdFromDB($mimeType) { - $sql = 'SELECT `id` FROM `*PREFIX*mimetypes` WHERE mimetype = ?'; + $sql = 'SELECT `id` FROM `*PREFIX*mimetypes` WHERE `mimetype` = ?'; $results = \OC_DB::executeAudited($sql, array($mimeType)); $result = $results->fetchOne(); if ($result) {