From 9adff4f7f52df660ec066b896161e46abe7a9d5a Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Fri, 22 Nov 2013 15:53:56 +0100 Subject: [PATCH 1/8] Change OC_Migrate from MDB2 to Doctrine --- lib/private/db.php | 2 +- lib/private/migrate.php | 96 +++++-------------------------- lib/private/migration/content.php | 35 ++++------- 3 files changed, 26 insertions(+), 107 deletions(-) diff --git a/lib/private/db.php b/lib/private/db.php index 1e5d12649d..237925a592 100644 --- a/lib/private/db.php +++ b/lib/private/db.php @@ -44,7 +44,7 @@ class OC_DB { /** * @var \OC\DB\Connection $connection */ - static private $connection; //the prefered connection to use, only Doctrine + static private $connection; //the preferred connection to use, only Doctrine static private $prefix=null; static private $type=null; diff --git a/lib/private/migrate.php b/lib/private/migrate.php index 0b31917740..b45848fbcb 100644 --- a/lib/private/migrate.php +++ b/lib/private/migrate.php @@ -38,7 +38,7 @@ class OC_Migrate{ // Array of temp files to be deleted after zip creation static private $tmpfiles=array(); // Holds the db object - static private $MDB2=false; + static private $migration_database=false; // Schema db object static private $schema=false; // Path to the sqlite db @@ -131,7 +131,7 @@ class OC_Migrate{ if( !self::connectDB() ) { return json_encode( array( 'success' => false ) ); } - self::$content = new OC_Migration_Content( self::$zip, self::$MDB2 ); + self::$content = new OC_Migration_Content( self::$zip, self::$migration_database ); // Export the app info $exportdata = self::exportAppData(); // Add the data dir to the zip @@ -358,24 +358,6 @@ class OC_Migrate{ return $to; } - /** - * @brief connects to a MDB2 database scheme - * @returns bool - */ - static private function connectScheme() { - // We need a mdb2 database connection - self::$MDB2->loadModule( 'Manager' ); - self::$MDB2->loadModule( 'Reverse' ); - - // Connect if this did not happen before - if( !self::$schema ) { - require_once 'MDB2/Schema.php'; - self::$schema=MDB2_Schema::factory( self::$MDB2 ); - } - - return true; - } - /** * @brief creates a migration.db in the users data dir with their app data in * @return bool whether operation was successfull @@ -463,47 +445,15 @@ class OC_Migrate{ return false; } // Already connected - if(!self::$MDB2) { - require_once 'MDB2.php'; - + if(!self::$migration_database) { $datadir = OC_Config::getValue( "datadirectory", OC::$SERVERROOT."/data" ); - - // DB type - if( class_exists( 'SQLite3' ) ) { - $dbtype = 'sqlite3'; - } else if( is_callable( 'sqlite_open' ) ) { - $dbtype = 'sqlite'; - } else { - OC_Log::write( 'migration', 'SQLite not found', OC_Log::ERROR ); - return false; - } - - // Prepare options array - $options = array( - 'portability' => MDB2_PORTABILITY_ALL & (!MDB2_PORTABILITY_FIX_CASE), - 'log_line_break' => '
', - 'idxname_format' => '%s', - 'debug' => true, - 'quote_identifier' => true - ); - $dsn = array( - 'phptype' => $dbtype, - 'database' => self::$dbpath, - 'mode' => '0644' + $connectionParams = array( + 'path' => self::$dbpath, + 'driver' => 'pdo_sqlite', ); // Try to establish connection - self::$MDB2 = MDB2::factory( $dsn, $options ); - // Die if we could not connect - if( PEAR::isError( self::$MDB2 ) ) { - die( self::$MDB2->getMessage() ); - OC_Log::write( 'migration', 'Failed to create/connect to migration.db', OC_Log::FATAL ); - OC_Log::write( 'migration', self::$MDB2->getUserInfo(), OC_Log::FATAL ); - OC_Log::write( 'migration', self::$MDB2->getMessage(), OC_Log::FATAL ); - return false; - } - // We always, really always want associative arrays - self::$MDB2->setFetchMode(MDB2_FETCHMODE_ASSOC); + self::$migration_database = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); } return true; @@ -515,10 +465,7 @@ class OC_Migrate{ * @return bool whether the operation was successful */ static private function createAppTables( $appid ) { - - if( !self::connectScheme() ) { - return false; - } + $schema_manager = new OC\DB\MDB2SchemaManager(self::$migration_database); // There is a database.xml file $content = file_get_contents(OC_App::getAppPath($appid) . '/appinfo/database.xml' ); @@ -538,29 +485,16 @@ class OC_Migrate{ file_put_contents( $file2, $content ); // Try to create tables - $definition = self::$schema->parseDatabaseDefinitionFile( $file2 ); - - unlink( $file2 ); - - // Die in case something went wrong - if( $definition instanceof MDB2_Schema_Error ) { - OC_Log::write( 'migration', 'Failed to parse database.xml for: '.$appid, OC_Log::FATAL ); - OC_Log::write( 'migration', $definition->getMessage().': '.$definition->getUserInfo(), OC_Log::FATAL ); - return false; - } - - $definition['overwrite'] = true; - - $ret = self::$schema->createDatabase( $definition ); - - // Die in case something went wrong - if( $ret instanceof MDB2_Error ) { + try { + $schema_manager->createDbFromStructure($file2); + } catch(Exception $e) { + unlink( $file2 ); OC_Log::write( 'migration', 'Failed to create tables for: '.$appid, OC_Log::FATAL ); - OC_Log::write( 'migration', $ret->getMessage().': '.$ret->getUserInfo(), OC_Log::FATAL ); + OC_Log::write( 'migration', $e->getMessage(), OC_Log::FATAL ); return false; } - return $tables; + return $tables; } /** @@ -646,7 +580,7 @@ class OC_Migrate{ if( !self::connectDB( $db ) ) { return false; } - $content = new OC_Migration_Content( self::$zip, self::$MDB2 ); + $content = new OC_Migration_Content( self::$zip, self::$migration_database ); $provider->setData( self::$uid, $content, $info ); // Then do the import if( !$appsstatus[$id] = $provider->import( $info->apps->$id, $importinfo ) ) { diff --git a/lib/private/migration/content.php b/lib/private/migration/content.php index 4413d72273..389dd8fc74 100644 --- a/lib/private/migration/content.php +++ b/lib/private/migration/content.php @@ -27,7 +27,7 @@ class OC_Migration_Content{ private $zip=false; - // Holds the MDB2 object + // Holds the database object private $db=null; // Holds an array of tmpfiles to delete after zip creation private $tmpfiles=array(); @@ -35,7 +35,7 @@ class OC_Migration_Content{ /** * @brief sets up the * @param $zip ZipArchive object - * @param optional $db a MDB2 database object (required for exporttype user) + * @param optional $db a database object (required for exporttype user) * @return bool */ public function __construct( $zip, $db=null ) { @@ -64,16 +64,7 @@ class OC_Migration_Content{ // Optimize the query $query = $this->db->prepare( $query ); - // Die if we have an error (error means: bad query, not 0 results!) - if( PEAR::isError( $query ) ) { - $entry = 'DB Error: "'.$query->getMessage().'"
'; - $entry .= 'Offending command was: '.$query.'
'; - OC_Log::write( 'migration', $entry, OC_Log::FATAL ); - return false; - } else { - return $query; - } - + return $query; } /** @@ -156,20 +147,14 @@ class OC_Migration_Content{ $sql .= $valuessql . " )"; // Make the query $query = $this->prepare( $sql ); - if( !$query ) { - OC_Log::write( 'migration', 'Invalid sql produced: '.$sql, OC_Log::FATAL ); - return false; - exit(); + $query->execute( $values ); + // Do we need to return some values? + if( array_key_exists( 'idcol', $options ) ) { + // Yes we do + $return[] = $row[$options['idcol']]; } else { - $query->execute( $values ); - // Do we need to return some values? - if( array_key_exists( 'idcol', $options ) ) { - // Yes we do - $return[] = $row[$options['idcol']]; - } else { - // Take a guess and return the first field :) - $return[] = reset($row); - } + // Take a guess and return the first field :) + $return[] = reset($row); } $fields = ''; $values = ''; From 14332681fc541aca936dcbd6baf7de220734815f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Fri, 22 Nov 2013 17:18:06 +0100 Subject: [PATCH 2/8] ref 3rdparty branch without MDB2 code --- 3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty b/3rdparty index 42efd96628..daf25846db 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit 42efd966284debadf83b761367e529bc45f806d6 +Subproject commit daf25846dbb8a326f4801dc3310cd2b8ab1f84b9 From 34fcf1e9d0b046afb35f3033cf8c3bd4a4c5ca62 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Fri, 20 Dec 2013 12:09:19 +0100 Subject: [PATCH 3/8] Wrap the import/export db in a wrapper to make it compatible with the old style --- lib/private/migrate.php | 3 +++ lib/private/migration/content.php | 1 + 2 files changed, 4 insertions(+) diff --git a/lib/private/migrate.php b/lib/private/migrate.php index b45848fbcb..5354e9d338 100644 --- a/lib/private/migrate.php +++ b/lib/private/migrate.php @@ -451,6 +451,9 @@ class OC_Migrate{ 'path' => self::$dbpath, 'driver' => 'pdo_sqlite', ); + $connectionParams['adapter'] = '\OC\DB\AdapterSqlite'; + $connectionParams['wrapperClass'] = 'OC\DB\Connection'; + $connectionParams['tablePrefix'] = ''; // Try to establish connection self::$migration_database = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); diff --git a/lib/private/migration/content.php b/lib/private/migration/content.php index 389dd8fc74..d6eee40786 100644 --- a/lib/private/migration/content.php +++ b/lib/private/migration/content.php @@ -63,6 +63,7 @@ class OC_Migration_Content{ // Optimize the query $query = $this->db->prepare( $query ); + $query = new OC_DB_StatementWrapper($query, false); return $query; } From ac85dea267261a58ae815c257098a3ae6ed42b80 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Fri, 20 Dec 2013 12:22:49 +0100 Subject: [PATCH 4/8] Fix migration import of user files --- lib/private/migrate.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/migrate.php b/lib/private/migrate.php index 5354e9d338..2d2d37795f 100644 --- a/lib/private/migrate.php +++ b/lib/private/migrate.php @@ -257,7 +257,7 @@ class OC_Migrate{ $userfolder = $extractpath . $json->exporteduser; $newuserfolder = $datadir . '/' . self::$uid; foreach(scandir($userfolder) as $file){ - if($file !== '.' && $file !== '..' && is_dir($file)) { + if($file !== '.' && $file !== '..' && is_dir($userfolder.'/'.$file)) { $file = str_replace(array('/', '\\'), '', $file); // Then copy the folder over From 528ff25edecfbe47d11e5e570746ff35dc547d85 Mon Sep 17 00:00:00 2001 From: tomneedham Date: Tue, 7 Jan 2014 00:38:27 +0000 Subject: [PATCH 5/8] Add WIP unit test for migration --- tests/lib/migrate.php | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/lib/migrate.php diff --git a/tests/lib/migrate.php b/tests/lib/migrate.php new file mode 100644 index 0000000000..5c6c9ebe7b --- /dev/null +++ b/tests/lib/migrate.php @@ -0,0 +1,32 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +class Test_Migrate extends PHPUnit_Framework_TestCase { + + public $users; + + public function testUserSelfExport(){ + // Create a user + $user = uniqid(); + $u = new OC_User(); + $u->createUser($user, 'password'); + $this->users[] = $user; + $exportLocation = \OC_Migrate::export($user); + } + + public function tearDown() { + $u = new OC_User(); + foreach($this->users as $user) { + $u->deleteUser($user); + } + } + + + + +} From 049e03c2b92e10ef94d1f7bc900e74f078a56a9b Mon Sep 17 00:00:00 2001 From: tomneedham Date: Mon, 17 Feb 2014 10:58:27 +0000 Subject: [PATCH 6/8] Finish basic testing of user export --- tests/lib/migrate.php | 69 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/tests/lib/migrate.php b/tests/lib/migrate.php index 5c6c9ebe7b..4b20173ba5 100644 --- a/tests/lib/migrate.php +++ b/tests/lib/migrate.php @@ -9,14 +9,68 @@ class Test_Migrate extends PHPUnit_Framework_TestCase { public $users; + public $tmpfiles = array(); - public function testUserSelfExport(){ + /** + * @breif Generates a test user and sets up their file system + * @return string the test users id + */ + public function generateUser() { + $username = uniqid(); + \OC_User::createUser($username, 'password'); + \OC_Util::tearDownFS(); + \OC_User::setUserId(''); + \OC\Files\Filesystem::tearDown(); + \OC_Util::setupFS($username); + $this->users[] = $username; + return $username; + } + + /** + * @breif validates an export for a user + * @breif checks for existence of export_info.json and file folder + * @param string $exportedUser the user that was exported + * @param string $path the path to the .zip export + */ + public function validateUserExport($exportedBy, $exportedUser, $path) { + $this->assertTrue(file_exists($path)); + // Extract + $extract = get_temp_dir() . '/oc_import_' . uniqid(); + //mkdir($extract); + $this->tmpfiles[] = $extract; + $zip = new ZipArchive; + $zip->open($path); + $zip->extractTo($extract); + $zip->close(); + $this->assertTrue(file_exists($extract.'/export_info.json')); + $exportInfo = file_get_contents($extract.'/export_info.json'); + $exportInfo = json_decode($exportInfo); + $this->assertNotNull($exportInfo); + $this->assertEquals($exportedUser, $exportInfo->exporteduser); + $this->assertEquals($exportedBy, $exportInfo->exportedby); + $this->assertTrue(file_exists($extract.'/'.$exportedUser.'/files')); + } + + public function testUserSelfExport() { // Create a user - $user = uniqid(); - $u = new OC_User(); - $u->createUser($user, 'password'); - $this->users[] = $user; - $exportLocation = \OC_Migrate::export($user); + $user = $this->generateUser(); + \OC_User::setUserId($user); + $export = \OC_Migrate::export($user); + // Check it succeeded and exists + $this->assertTrue(json_decode($export)->success); + // Validate the export + $this->validateUserExport($user, $user, json_decode($export)->data); + } + + public function testUserOtherExport() { + $user = $this->generateUser(); + $user2 = $this->generateUser(); + \OC_User::setUserId($user2); + $export = \OC_Migrate::export($user); + // Check it succeeded and exists + $this->assertTrue(json_decode($export)->success); + // Validate the export + $this->validateUserExport($user2, $user, json_decode($export)->data); } public function tearDown() { @@ -24,6 +78,9 @@ class Test_Migrate extends PHPUnit_Framework_TestCase { foreach($this->users as $user) { $u->deleteUser($user); } + foreach($this->tmpfiles as $file) { + \OC_Helper::rmdirr($file); + } } From 589a7b220b7d2ec932dbda6419577bfdc0ed9f86 Mon Sep 17 00:00:00 2001 From: tomneedham Date: Tue, 18 Feb 2014 16:23:16 +0000 Subject: [PATCH 7/8] Fix poor spelling --- tests/lib/migrate.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/lib/migrate.php b/tests/lib/migrate.php index 4b20173ba5..39a9bfc8d5 100644 --- a/tests/lib/migrate.php +++ b/tests/lib/migrate.php @@ -12,7 +12,7 @@ class Test_Migrate extends PHPUnit_Framework_TestCase { public $tmpfiles = array(); /** - * @breif Generates a test user and sets up their file system + * @brief Generates a test user and sets up their file system * @return string the test users id */ public function generateUser() { @@ -27,8 +27,8 @@ class Test_Migrate extends PHPUnit_Framework_TestCase { } /** - * @breif validates an export for a user - * @breif checks for existence of export_info.json and file folder + * @brief validates an export for a user + * @brief checks for existence of export_info.json and file folder * @param string $exportedUser the user that was exported * @param string $path the path to the .zip export */ From 0beaeed713977bcca0eb9da996af712526ee69d2 Mon Sep 17 00:00:00 2001 From: tomneedham Date: Tue, 18 Feb 2014 16:28:04 +0000 Subject: [PATCH 8/8] Remove unused variables --- lib/private/migrate.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/private/migrate.php b/lib/private/migrate.php index 2d2d37795f..948b3d4572 100644 --- a/lib/private/migrate.php +++ b/lib/private/migrate.php @@ -35,12 +35,8 @@ class OC_Migrate{ static private $zip=false; // Stores the type of export static private $exporttype=false; - // Array of temp files to be deleted after zip creation - static private $tmpfiles=array(); // Holds the db object static private $migration_database=false; - // Schema db object - static private $schema=false; // Path to the sqlite db static private $dbpath=false; // Holds the path to the zip file