Skip to content

Commit 273cd1d

Browse files
artongeartonge
committed
Move oc_file_metadata.metadata migration to a background job
Signed-off-by: Louis Chemineau <louis@chmn.me>
1 parent 0386641 commit 273cd1d

File tree

5 files changed

+176
-28
lines changed

5 files changed

+176
-28
lines changed

‎core/BackgroundJobs/MetadataMigrationJob.php‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* @copyright Copyright (c) 2023 Louis Chemineau <louis@chmn.me>
6+
*
7+
* @author Louis Chemineau <louis@chmn.me>
8+
*
9+
* @license AGPL-3.0-or-later
10+
*
11+
* This program is free software: you can redistribute it and/or modify
12+
* it under the terms of the GNU Affero General Public License as
13+
* published by the Free Software Foundation, either version 3 of the
14+
* License, or (at your option) any later version.
15+
*
16+
* This program is distributed in the hope that it will be useful,
17+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
18+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+
* GNU Affero General Public License for more details.
20+
*
21+
* You should have received a copy of the GNU Affero General Public License
22+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
23+
*
24+
*/
25+
26+
namespace OC\Core\BackgroundJobs;
27+
28+
use OCP\AppFramework\Utility\ITimeFactory;
29+
use OCP\BackgroundJob\IJobList;
30+
use OCP\BackgroundJob\TimedJob;
31+
use OCP\DB\QueryBuilder\IQueryBuilder;
32+
use OCP\IDBConnection;
33+
34+
// Migrate oc_file_metadata.metadata to oc_file_metadata.value.
35+
// This was previously done in a migration, but it is taking to much time in large instances.
36+
// This job will progressively migrate the data 1 hour per night every night.
37+
// Once done, it will remove itself from the job list and drop oc_file_metadata.metadata.
38+
class MetadataMigrationJob extends TimedJob {
39+
public function __construct(
40+
ITimeFactory $time,
41+
private IDBConnection $db,
42+
private IJobList $jobList,
43+
) {
44+
parent::__construct($time);
45+
46+
$this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE);
47+
$this->setInterval(24 * 3600);
48+
}
49+
50+
protected function run(mixed $argument): void {
51+
if (!$this->db->createSchema()->getTable('oc_file_metadata')->hasColumn('metadata')) {
52+
return;
53+
}
54+
55+
$updateQuery = $this->db->getQueryBuilder();
56+
$updateQuery->update('file_metadata')
57+
->set('value', $updateQuery->createParameter('value'))
58+
->set('metadata', $updateQuery->createParameter('metadata'))
59+
->where($updateQuery->expr()->eq('id', $updateQuery->createParameter('id')))
60+
->andWhere($updateQuery->expr()->eq('group_name', $updateQuery->createParameter('group_name')));
61+
62+
$selectQuery = $this->db->getQueryBuilder();
63+
$selectQuery->select('id', 'group_name', 'metadata')
64+
->from('file_metadata')
65+
->where($selectQuery->expr()->neq('metadata', $selectQuery->createNamedParameter(''), IQueryBuilder::PARAM_STR))
66+
->orderBy('id', 'ASC')
67+
->setMaxResults(1000);
68+
69+
$movedRows = 0;
70+
$startTime = time();
71+
72+
do {
73+
// Stop if execution time is more than one hour.
74+
if (time() - $startTime > 3600) {
75+
return;
76+
}
77+
$movedRows = $this->chunkedCopying($updateQuery, $selectQuery);
78+
} while ($movedRows !== 0);
79+
80+
81+
$this->jobList->remove(MetadataMigrationJob::class);
82+
}
83+
84+
protected function chunkedCopying(IQueryBuilder $updateQuery, IQueryBuilder $selectQuery): int {
85+
$this->db->beginTransaction();
86+
87+
$results = $selectQuery->executeQuery();
88+
89+
while ($row = $results->fetch()) {
90+
$updateQuery
91+
->setParameter('id', (int)$row['id'])
92+
->setParameter('group_name', $row['group_name'])
93+
->setParameter('value', $row['metadata'])
94+
->setParameter('metadata', '')
95+
->executeStatement();
96+
}
97+
98+
$results->closeCursor();
99+
$this->db->commit();
100+
101+
return $results->rowCount();
102+
}
103+
}

‎core/Migrations/Version27000Date20230309104325.php‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -72,19 +72,19 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
7272
* @param array $options
7373
* @return void
7474
*/
75-
public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
76-
/** @var ISchemaWrapper $schema */
77-
$schema = $schemaClosure();
78-
$metadataTable = $schema->getTable('file_metadata');
75+
// public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
76+
// /** @var ISchemaWrapper $schema */
77+
// $schema = $schemaClosure();
78+
// $metadataTable = $schema->getTable('file_metadata');
7979

80-
if (!$metadataTable->hasColumn('metadata')) {
81-
return;
82-
}
80+
// if (!$metadataTable->hasColumn('metadata')) {
81+
// return;
82+
// }
8383

84-
$this->connection
85-
->getQueryBuilder()
86-
->update('file_metadata')
87-
->set('value', 'metadata')
88-
->executeStatement();
89-
}
84+
// $this->connection
85+
// ->getQueryBuilder()
86+
// ->update('file_metadata')
87+
// ->set('value', 'metadata')
88+
// ->executeStatement();
89+
// }
9090
}

‎core/Migrations/Version27000Date20230309104802.php‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,14 @@ class Version27000Date20230309104802 extends SimpleMigrationStep {
4343
* @return null|ISchemaWrapper
4444
*/
4545
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
46-
/** @var ISchemaWrapper $schema */
47-
$schema = $schemaClosure();
48-
$metadataTable = $schema->getTable('file_metadata');
46+
// /** @var ISchemaWrapper $schema */
47+
// $schema = $schemaClosure();
48+
// $metadataTable = $schema->getTable('file_metadata');
4949

50-
if ($metadataTable->hasColumn('metadata')) {
51-
$metadataTable->dropColumn('metadata');
52-
return $schema;
53-
}
50+
// if ($metadataTable->hasColumn('metadata')) {
51+
// $metadataTable->dropColumn('metadata');
52+
// return $schema;
53+
// }
5454

5555
return null;
5656
}

‎lib/private/Repair.php‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,19 +34,14 @@
3434
*/
3535
namespace OC;
3636

37-
use OC\Repair\AddRemoveOldTasksBackgroundJob;
38-
use OC\Repair\CleanUpAbandonedApps;
39-
use OCP\AppFramework\QueryException;
40-
use OCP\AppFramework\Utility\ITimeFactory;
41-
use OCP\Collaboration\Resources\IManager;
42-
use OCP\EventDispatcher\IEventDispatcher;
43-
use OCP\Migration\IOutput;
44-
use OCP\Migration\IRepairStep;
4537
use OC\DB\Connection;
4638
use OC\DB\ConnectionAdapter;
4739
use OC\Repair\AddBruteForceCleanupJob;
4840
use OC\Repair\AddCleanupUpdaterBackupsJob;
41+
use OC\Repair\AddMetadataMigrationJob;
42+
use OC\Repair\AddRemoveOldTasksBackgroundJob;
4943
use OC\Repair\CleanTags;
44+
use OC\Repair\CleanUpAbandonedApps;
5045
use OC\Repair\ClearFrontendCaches;
5146
use OC\Repair\ClearGeneratedAvatarCache;
5247
use OC\Repair\Collation;
@@ -86,6 +81,12 @@
8681
use OC\Repair\RepairMimeTypes;
8782
use OC\Repair\SqliteAutoincrement;
8883
use OC\Template\JSCombiner;
84+
use OCP\AppFramework\QueryException;
85+
use OCP\AppFramework\Utility\ITimeFactory;
86+
use OCP\Collaboration\Resources\IManager;
87+
use OCP\EventDispatcher\IEventDispatcher;
88+
use OCP\Migration\IOutput;
89+
use OCP\Migration\IRepairStep;
8990
use Psr\Log\LoggerInterface;
9091
use Throwable;
9192

@@ -212,6 +213,7 @@ public static function getRepairSteps(): array {
212213
\OCP\Server::get(CleanUpAbandonedApps::class),
213214
\OCP\Server::get(AddMissingSecretJob::class),
214215
\OCP\Server::get(AddRemoveOldTasksBackgroundJob::class),
216+
\OCP\Server::get(AddMetadataMigrationJob::class),
215217
];
216218
}
217219

‎lib/private/Repair/AddMetadataMigrationJob.php‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?php
2+
/**
3+
* @copyright Copyright (c) 2023 Louis Chmn <louis@chmn.me>
4+
*
5+
* @author Louis Chmn <louis@chmn.me>
6+
*
7+
* @license GNU AGPL version 3 or any later version
8+
*
9+
* This program is free software: you can redistribute it and/or modify
10+
* it under the terms of the GNU Affero General Public License as
11+
* published by the Free Software Foundation, either version 3 of the
12+
* License, or (at your option) any later version.
13+
*
14+
* This program is distributed in the hope that it will be useful,
15+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
* GNU Affero General Public License for more details.
18+
*
19+
* You should have received a copy of the GNU Affero General Public License
20+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
21+
*
22+
*/
23+
namespace OC\Repair;
24+
25+
use OC\Core\BackgroundJobs\MetadataMigrationJob;
26+
use OCP\BackgroundJob\IJobList;
27+
use OCP\Migration\IOutput;
28+
use OCP\Migration\IRepairStep;
29+
30+
class AddMetadataMigrationJob implements IRepairStep {
31+
public function __construct(
32+
private IJobList $jobList,
33+
) {
34+
}
35+
36+
public function getName() {
37+
return 'Queue a job to migrate the file_metadata table';
38+
}
39+
40+
public function run(IOutput $output) {
41+
$this->jobList->add(MetadataMigrationJob::class);
42+
}
43+
}

0 commit comments

Comments
 (0)