Update reshares in batches as not to run out of memory
This commit is contained in:
parent
dc8d43575f
commit
b85770d636
|
@ -23,6 +23,7 @@ namespace OCA\Files_Sharing;
|
||||||
|
|
||||||
use Doctrine\DBAL\Connection;
|
use Doctrine\DBAL\Connection;
|
||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
use OC\Cache\CappedMemoryCache;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Class Migration
|
* Class Migration
|
||||||
|
@ -43,6 +44,9 @@ class Migration {
|
||||||
|
|
||||||
public function __construct(IDBConnection $connection) {
|
public function __construct(IDBConnection $connection) {
|
||||||
$this->connection = $connection;
|
$this->connection = $connection;
|
||||||
|
|
||||||
|
// We cache up to 10k share items (~20MB)
|
||||||
|
$this->shareCache = new CappedMemoryCache(10000);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -50,8 +54,19 @@ class Migration {
|
||||||
* upgrade from oC 8.2 to 9.0 with the new sharing
|
* upgrade from oC 8.2 to 9.0 with the new sharing
|
||||||
*/
|
*/
|
||||||
public function removeReShares() {
|
public function removeReShares() {
|
||||||
$reShares = $this->getAllReShares();
|
|
||||||
$this->shareCache = $reShares;
|
while(true) {
|
||||||
|
$reShares = $this->getReShares(1000);
|
||||||
|
|
||||||
|
if (empty($reShares)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update the cache
|
||||||
|
foreach($reShares as $reShare) {
|
||||||
|
$this->shareCache[$reShare['id']] = $reShare;
|
||||||
|
}
|
||||||
|
|
||||||
$owners = [];
|
$owners = [];
|
||||||
foreach ($reShares as $share) {
|
foreach ($reShares as $share) {
|
||||||
$owners[$share['id']] = [
|
$owners[$share['id']] = [
|
||||||
|
@ -59,8 +74,13 @@ class Migration {
|
||||||
'initiator' => $share['uid_owner']
|
'initiator' => $share['uid_owner']
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->updateOwners($owners);
|
$this->updateOwners($owners);
|
||||||
|
|
||||||
|
//Clear the cache of the shares we just updated so we have more room
|
||||||
|
foreach($owners as $id => $owner) {
|
||||||
|
unset($this->shareCache[$id]);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -84,11 +104,12 @@ class Migration {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* get all re-shares from the database
|
* Get $n re-shares from the database
|
||||||
*
|
*
|
||||||
|
* @param int $n The max number of shares to fetch
|
||||||
* @return array
|
* @return array
|
||||||
*/
|
*/
|
||||||
private function getAllReShares() {
|
private function getReShares($n = 1000) {
|
||||||
$query = $this->connection->getQueryBuilder();
|
$query = $this->connection->getQueryBuilder();
|
||||||
$query->select(['id', 'parent', 'uid_owner'])
|
$query->select(['id', 'parent', 'uid_owner'])
|
||||||
->from($this->table)
|
->from($this->table)
|
||||||
|
@ -110,7 +131,9 @@ class Migration {
|
||||||
Connection::PARAM_STR_ARRAY
|
Connection::PARAM_STR_ARRAY
|
||||||
)
|
)
|
||||||
))
|
))
|
||||||
->andWhere($query->expr()->isNotNull('parent'));
|
->andWhere($query->expr()->isNotNull('parent'))
|
||||||
|
->orderBy('id', 'asc')
|
||||||
|
->setMaxResults($n);
|
||||||
$result = $query->execute();
|
$result = $query->execute();
|
||||||
$shares = $result->fetchAll();
|
$shares = $result->fetchAll();
|
||||||
$result->closeCursor();
|
$result->closeCursor();
|
||||||
|
@ -159,7 +182,8 @@ class Migration {
|
||||||
->set('parent', $query->createNamedParameter(null))
|
->set('parent', $query->createNamedParameter(null))
|
||||||
->set('uid_owner', $query->createNamedParameter($owner['owner']))
|
->set('uid_owner', $query->createNamedParameter($owner['owner']))
|
||||||
->set('uid_initiator', $query->createNamedParameter($owner['initiator']))
|
->set('uid_initiator', $query->createNamedParameter($owner['initiator']))
|
||||||
->where($query->expr()->eq('id', $query->createNamedParameter($id)))->execute();
|
->where($query->expr()->eq('id', $query->createNamedParameter($id)))
|
||||||
|
->execute();
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->connection->commit();
|
$this->connection->commit();
|
||||||
|
|
|
@ -246,4 +246,63 @@ class MigrationTest extends TestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function test100kDeepReshares() {
|
||||||
|
$parent = null;
|
||||||
|
for ($i = 0; $i < 10; $i++) {
|
||||||
|
$query = $this->connection->getQueryBuilder();
|
||||||
|
$query->insert($this->table)
|
||||||
|
->values(
|
||||||
|
[
|
||||||
|
'share_type' => $query->createParameter('share_type'),
|
||||||
|
'share_with' => $query->createParameter('share_with'),
|
||||||
|
'uid_owner' => $query->createParameter('uid_owner'),
|
||||||
|
'uid_initiator' => $query->createParameter('uid_initiator'),
|
||||||
|
'parent' => $query->createParameter('parent'),
|
||||||
|
'item_type' => $query->createParameter('item_type'),
|
||||||
|
'item_source' => $query->createParameter('item_source'),
|
||||||
|
'item_target' => $query->createParameter('item_target'),
|
||||||
|
'file_source' => $query->createParameter('file_source'),
|
||||||
|
'file_target' => $query->createParameter('file_target'),
|
||||||
|
'permissions' => $query->createParameter('permissions'),
|
||||||
|
'stime' => $query->createParameter('stime'),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
->setParameter('share_type', \OCP\Share::SHARE_TYPE_USER)
|
||||||
|
->setParameter('share_with', 'user'.($i+1))
|
||||||
|
->setParameter('uid_owner', 'user'.($i))
|
||||||
|
->setParameter('uid_initiator', '')
|
||||||
|
->setParameter('parent', $parent)
|
||||||
|
->setParameter('item_type', 'file')
|
||||||
|
->setParameter('item_source', '2')
|
||||||
|
->setParameter('item_target', '/2')
|
||||||
|
->setParameter('file_source', 2)
|
||||||
|
->setParameter('file_target', '/foobar')
|
||||||
|
->setParameter('permissions', 31)
|
||||||
|
->setParameter('stime', time());
|
||||||
|
|
||||||
|
$this->assertSame(1, $query->execute());
|
||||||
|
$parent = $query->getLastInsertId();
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->migration->removeReShares();
|
||||||
|
|
||||||
|
$qb = $this->connection->getQueryBuilder();
|
||||||
|
|
||||||
|
$stmt = $qb->select('id', 'share_with', 'uid_owner', 'uid_initiator', 'parent')
|
||||||
|
->from('share')
|
||||||
|
->orderBy('id', 'asc')
|
||||||
|
->execute();
|
||||||
|
|
||||||
|
$i = 0;
|
||||||
|
while($share = $stmt->fetch()) {
|
||||||
|
$this->assertEquals('user'.($i+1), $share['share_with']);
|
||||||
|
if ($i !== 0) {
|
||||||
|
$this->assertEquals('user' . ($i), $share['uid_initiator']);
|
||||||
|
$this->assertEquals('user0', $share['uid_owner']);
|
||||||
|
}
|
||||||
|
$this->assertEquals(null, $share['parent']);
|
||||||
|
$i++;
|
||||||
|
}
|
||||||
|
$stmt->closeCursor();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue