Skip to content

Commit c31f611

Browse files
abnegateabnegate
committed
Fix validation
1 parent aee612a commit c31f611

File tree

6 files changed

+607
-91
lines changed

6 files changed

+607
-91
lines changed

‎bin/tasks/operators.php‎

Lines changed: 73 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,11 @@
88
*
99
* @example
1010
* docker compose exec tests bin/operators --adapter=mariadb --iterations=1000
11-
* docker compose exec tests bin/operators --adapter=postgres --iterations=1000
12-
* docker compose exec tests bin/operators --adapter=sqlite --iterations=1000
11+
* docker compose exec tests bin/operators --adapter=postgres --iterations=1000 --seed=10000
12+
* docker compose exec tests bin/operators --adapter=sqlite --iterations=1000 --seed=5000
13+
*
14+
* The --seed parameter allows you to pre-populate the collection with a specified
15+
* number of documents to test how operators perform with varying amounts of existing data.
1316
*/
1417

1518
global $cli;
@@ -38,8 +41,9 @@
3841
->desc('Benchmark operator performance vs traditional read-modify-write')
3942
->param('adapter', '', new Text(0), 'Database adapter (mariadb, postgres, sqlite)')
4043
->param('iterations', 1000, new Integer(true), 'Number of iterations per test', true)
44+
->param('seed', 0, new Integer(true), 'Number of documents to pre-seed the collection with', true)
4145
->param('name', 'operator_benchmark_' . uniqid(), new Text(0), 'Name of test database', true)
42-
->action(function (string $adapter, int $iterations, string $name) {
46+
->action(function (string $adapter, int $iterations, int $seed, string $name) {
4347
$namespace = '_ns';
4448
$cache = new Cache(new NoCache());
4549

@@ -48,6 +52,7 @@
4852
Console::info("=============================================================");
4953
Console::info("Adapter: {$adapter}");
5054
Console::info("Iterations: {$iterations}");
55+
Console::info("Seed Documents: {$seed}");
5156
Console::info("Database: {$name}");
5257
Console::info("=============================================================\n");
5358

@@ -110,13 +115,13 @@
110115
->setNamespace($namespace);
111116

112117
// Setup test environment
113-
setupTestEnvironment($database, $name);
118+
setupTestEnvironment($database, $name, $seed);
114119

115120
// Run all benchmarks
116121
$results = runAllBenchmarks($database, $iterations);
117122

118123
// Display results
119-
displayResults($results, $adapter, $iterations);
124+
displayResults($results, $adapter, $iterations, $seed);
120125

121126
// Cleanup
122127
cleanup($database, $name);
@@ -133,7 +138,7 @@
133138
/**
134139
* Setup test environment with collections and sample data
135140
*/
136-
function setupTestEnvironment(Database $database, string $name): void
141+
function setupTestEnvironment(Database $database, string $name, int $seed): void
137142
{
138143
Console::info("Setting up test environment...");
139144

@@ -179,9 +184,69 @@ function setupTestEnvironment(Database $database, string $name): void
179184
$database->createAttribute('operators_test', 'created_at', Database::VAR_DATETIME, 0, false, null, false, false, null, [], ['datetime']);
180185
$database->createAttribute('operators_test', 'updated_at', Database::VAR_DATETIME, 0, false, null, false, false, null, [], ['datetime']);
181186

187+
// Seed documents if requested
188+
if ($seed > 0) {
189+
seedDocuments($database, $seed);
190+
}
191+
182192
Console::success("Test environment setup complete.\n");
183193
}
184194

195+
/**
196+
* Seed the collection with a specified number of documents
197+
*/
198+
function seedDocuments(Database $database, int $count): void
199+
{
200+
Console::info("Seeding {$count} documents...");
201+
202+
$batchSize = 100; // Insert in batches for better performance
203+
$batches = (int) ceil($count / $batchSize);
204+
205+
$seedStart = microtime(true);
206+
207+
for ($batch = 0; $batch < $batches; $batch++) {
208+
$docs = [];
209+
$remaining = min($batchSize, $count - ($batch * $batchSize));
210+
211+
for ($i = 0; $i < $remaining; $i++) {
212+
$docNum = ($batch * $batchSize) + $i;
213+
$docs[] = new Document([
214+
'$id' => 'seed_' . $docNum,
215+
'$permissions' => [
216+
Permission::read(Role::any()),
217+
Permission::update(Role::any()),
218+
],
219+
'counter' => rand(0, 1000),
220+
'score' => round(rand(0, 10000) / 100, 2),
221+
'multiplier' => round(rand(50, 200) / 100, 2),
222+
'divider' => round(rand(5000, 15000) / 100, 2),
223+
'modulo_val' => rand(50, 200),
224+
'power_val' => round(rand(100, 300) / 100, 2),
225+
'name' => 'seed_doc_' . $docNum,
226+
'text' => 'Seed text for document ' . $docNum,
227+
'description' => 'This is seed document ' . $docNum . ' with some foo bar baz content',
228+
'active' => (bool) rand(0, 1),
229+
'tags' => ['seed', 'tag' . ($docNum % 10), 'category' . ($docNum % 5)],
230+
'numbers' => [rand(1, 10), rand(11, 20), rand(21, 30)],
231+
'items' => ['item' . ($docNum % 3), 'item' . ($docNum % 7)],
232+
'created_at' => DateTime::now(),
233+
'updated_at' => DateTime::now(),
234+
]);
235+
}
236+
237+
// Bulk insert documents
238+
$database->createDocuments('operators_test', $docs);
239+
240+
// Show progress
241+
$progress = (($batch + 1) * $batchSize);
242+
$current = min($progress, $count);
243+
Console::log(" Seeded {$current}/{$count} documents...");
244+
}
245+
246+
$seedTime = microtime(true) - $seedStart;
247+
Console::success("Seeding completed in " . number_format($seedTime, 2) . "s\n");
248+
}
249+
185250
/**
186251
* Run all operator benchmarks
187252
*/
@@ -848,13 +913,14 @@ function benchmarkOperatorAcrossOperations(
848913
/**
849914
* Display formatted results table
850915
*/
851-
function displayResults(array $results, string $adapter, int $iterations): void
916+
function displayResults(array $results, string $adapter, int $iterations, int $seed): void
852917
{
853918
Console::info("\n=============================================================");
854919
Console::info(" BENCHMARK RESULTS");
855920
Console::info("=============================================================");
856921
Console::info("Adapter: {$adapter}");
857922
Console::info("Iterations per test: {$iterations}");
923+
Console::info("Seeded documents: {$seed}");
858924
Console::info("=============================================================\n");
859925

860926
// ==================================================================

‎src/Database/Database.php‎

Lines changed: 40 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -4952,20 +4952,7 @@ public function updateDocument(string $collection, string $id, Document $documen
49524952
}
49534953
$createdAt = $document->getCreatedAt();
49544954

4955-
// Extract operators from the document before merging
4956-
$documentArray = $document->getArrayCopy();
4957-
$extracted = Operator::extractOperators($documentArray);
4958-
$operators = $extracted['operators'];
4959-
$updates = $extracted['updates'];
4960-
4961-
$operatorValidator = new OperatorValidator($collection, $old);
4962-
foreach ($operators as $attribute => $operator) {
4963-
if (!$operatorValidator->isValid($operator)) {
4964-
throw new StructureException($operatorValidator->getDescription());
4965-
}
4966-
}
4967-
4968-
$document = \array_merge($old->getArrayCopy(), $updates);
4955+
$document = \array_merge($old->getArrayCopy(), $document->getArrayCopy());
49694956
$document['$collection'] = $old->getAttribute('$collection'); // Make sure user doesn't switch collection ID
49704957
$document['$createdAt'] = ($createdAt === null || !$this->preserveDates) ? $old->getCreatedAt() : $createdAt;
49714958

@@ -4989,8 +4976,11 @@ public function updateDocument(string $collection, string $id, Document $documen
49894976
$relationships[$relationship->getAttribute('key')] = $relationship;
49904977
}
49914978

4992-
if (!empty($operators)) {
4993-
$shouldUpdate = true;
4979+
foreach ($document as $key => $value) {
4980+
if (Operator::isOperator($value)) {
4981+
$shouldUpdate = true;
4982+
break;
4983+
}
49944984
}
49954985

49964986
// Compare if the document has any changes
@@ -5110,7 +5100,8 @@ public function updateDocument(string $collection, string $id, Document $documen
51105100
$this->adapter->getIdAttributeType(),
51115101
$this->adapter->getMinDateTime(),
51125102
$this->adapter->getMaxDateTime(),
5113-
$this->adapter->getSupportForAttributes()
5103+
$this->adapter->getSupportForAttributes(),
5104+
$old
51145105
);
51155106
if (!$structureValidator->isValid($document)) { // Make sure updated structure still apply collection rules (if any)
51165107
throw new StructureException($structureValidator->getDescription());
@@ -5120,22 +5111,24 @@ public function updateDocument(string $collection, string $id, Document $documen
51205111
$document = $this->silent(fn () => $this->updateDocumentRelationships($collection, $old, $document));
51215112
}
51225113

5123-
51245114
$document = $this->adapter->castingBefore($collection, $document);
51255115

5126-
// Re-add operators to document for adapter processing
5127-
foreach ($operators as $key => $operator) {
5128-
$document->setAttribute($key, $operator);
5129-
}
5130-
51315116
$this->adapter->updateDocument($collection, $id, $document, $skipPermissionsUpdate);
51325117

51335118
$document = $this->adapter->castingAfter($collection, $document);
51345119

51355120
$this->purgeCachedDocument($collection->getId(), $id);
51365121

51375122
// If operators were used, refetch document to get computed values
5138-
if (!empty($operators)) {
5123+
$hasOperators = false;
5124+
foreach ($document->getArrayCopy() as $value) {
5125+
if (Operator::isOperator($value)) {
5126+
$hasOperators = true;
5127+
break;
5128+
}
5129+
}
5130+
5131+
if ($hasOperators) {
51395132
$refetched = $this->refetchDocuments($collection, [$document]);
51405133
$document = $refetched[0];
51415134
}
@@ -5258,24 +5251,17 @@ public function updateDocuments(
52585251
applyDefaults: false
52595252
);
52605253

5261-
// Separate operators from regular updates for validation
5262-
$extracted = Operator::extractOperators($updates->getArrayCopy());
5263-
$operators = $extracted['operators'];
5264-
$regularUpdates = $extracted['updates'];
5265-
5266-
// Only validate regular updates, not operators
5267-
if (!empty($regularUpdates)) {
5268-
$validator = new PartialStructure(
5269-
$collection,
5270-
$this->adapter->getIdAttributeType(),
5271-
$this->adapter->getMinDateTime(),
5272-
$this->adapter->getMaxDateTime(),
5273-
$this->adapter->getSupportForAttributes()
5274-
);
5254+
$validator = new PartialStructure(
5255+
$collection,
5256+
$this->adapter->getIdAttributeType(),
5257+
$this->adapter->getMinDateTime(),
5258+
$this->adapter->getMaxDateTime(),
5259+
$this->adapter->getSupportForAttributes(),
5260+
null // No old document available in bulk updates
5261+
);
52755262

5276-
if (!$validator->isValid(new Document($regularUpdates))) {
5277-
throw new StructureException($validator->getDescription());
5278-
}
5263+
if (!$validator->isValid($updates)) {
5264+
throw new StructureException($validator->getDescription());
52795265
}
52805266

52815267
$originalLimit = $limit;
@@ -5311,17 +5297,8 @@ public function updateDocuments(
53115297
$currentPermissions = $updates->getPermissions();
53125298
sort($currentPermissions);
53135299

5314-
$this->withTransaction(function () use ($collection, $updates, &$batch, $currentPermissions, $operators) {
5300+
$this->withTransaction(function () use ($collection, $updates, &$batch, $currentPermissions) {
53155301
foreach ($batch as $index => $document) {
5316-
if (!empty($operators)) {
5317-
$operatorValidator = new OperatorValidator($collection, $document);
5318-
foreach ($operators as $attribute => $operator) {
5319-
if (!$operatorValidator->isValid($operator)) {
5320-
throw new StructureException($operatorValidator->getDescription());
5321-
}
5322-
}
5323-
}
5324-
53255302
$skipPermissionsUpdate = true;
53265303

53275304
if ($updates->offsetExists('$permissions')) {
@@ -5369,7 +5346,15 @@ public function updateDocuments(
53695346

53705347
$updates = $this->adapter->castingBefore($collection, $updates);
53715348

5372-
if (!empty($operators)) {
5349+
$hasOperators = false;
5350+
foreach ($updates->getArrayCopy() as $value) {
5351+
if (Operator::isOperator($value)) {
5352+
$hasOperators = true;
5353+
break;
5354+
}
5355+
}
5356+
5357+
if ($hasOperators) {
53735358
$batch = $this->refetchDocuments($collection, $batch);
53745359
}
53755360

@@ -6035,45 +6020,19 @@ public function upsertDocumentsWithIncrease(
60356020
}
60366021
}
60376022

6038-
// Extract operators for validation
6039-
$documentArray = $document->getArrayCopy();
6040-
$extracted = Operator::extractOperators($documentArray);
6041-
$operators = $extracted['operators'];
6042-
$regularUpdates = $extracted['updates'];
6043-
6044-
$operatorValidator = new OperatorValidator($collection, $old->isEmpty() ? null : $old);
6045-
foreach ($operators as $attribute => $operator) {
6046-
if (!$operatorValidator->isValid($operator)) {
6047-
throw new StructureException($operatorValidator->getDescription());
6048-
}
6049-
}
6050-
6051-
// Create a temporary document with only regular updates for encoding and validation
6052-
$tempDocument = new Document($regularUpdates);
6053-
$tempDocument->setAttribute('$id', $document->getId());
6054-
$tempDocument->setAttribute('$collection', $document->getAttribute('$collection'));
6055-
$tempDocument->setAttribute('$createdAt', $document->getAttribute('$createdAt'));
6056-
$tempDocument->setAttribute('$updatedAt', $document->getAttribute('$updatedAt'));
6057-
$tempDocument->setAttribute('$permissions', $document->getAttribute('$permissions'));
6058-
if ($this->adapter->getSharedTables()) {
6059-
$tempDocument->setAttribute('$tenant', $document->getAttribute('$tenant'));
6060-
}
6061-
6062-
$encodedTemp = $this->encode($collection, $tempDocument);
6063-
60646023
$validator = new Structure(
60656024
$collection,
60666025
$this->adapter->getIdAttributeType(),
60676026
$this->adapter->getMinDateTime(),
60686027
$this->adapter->getMaxDateTime(),
6069-
$this->adapter->getSupportForAttributes()
6028+
$this->adapter->getSupportForAttributes(),
6029+
$old->isEmpty() ? null : $old
60706030
);
60716031

6072-
if (!$validator->isValid($encodedTemp)) {
6032+
if (!$validator->isValid($document)) {
60736033
throw new StructureException($validator->getDescription());
60746034
}
60756035

6076-
// Now encode the full document with operators for the adapter
60776036
$document = $this->encode($collection, $document);
60786037

60796038
if (!$old->isEmpty()) {

‎src/Database/Validator/Operator.php‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,12 @@ public function getDescription(): string
6060
public function isValid($value): bool
6161
{
6262
if (!$value instanceof DatabaseOperator) {
63-
$this->message = 'Value must be an instance of Operator';
64-
return false;
63+
try {
64+
$value = DatabaseOperator::parse($value);
65+
} catch (\Throwable $e) {
66+
$this->message = 'Invalid operator: ' . $e->getMessage();
67+
return false;
68+
}
6569
}
6670

6771
$method = $value->getMethod();

‎src/Database/Validator/Structure.php‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
use Utopia\Database\Database;
88
use Utopia\Database\Document;
99
use Utopia\Database\Exception as DatabaseException;
10+
use Utopia\Database\Operator;
1011
use Utopia\Database\Validator\Datetime as DatetimeValidator;
12+
use Utopia\Database\Validator\Operator as OperatorValidator;
1113
use Utopia\Validator;
1214
use Utopia\Validator\Boolean;
1315
use Utopia\Validator\FloatValidator;
@@ -106,7 +108,8 @@ public function __construct(
106108
private readonly string $idAttributeType,
107109
private readonly \DateTime $minAllowedDate = new \DateTime('0000-01-01'),
108110
private readonly \DateTime $maxAllowedDate = new \DateTime('9999-12-31'),
109-
private bool $supportForAttributes = true
111+
private bool $supportForAttributes = true,
112+
private readonly ?Document $currentDocument = null
110113
) {
111114
}
112115

@@ -305,6 +308,18 @@ protected function checkForUnknownAttributes(array $structure, array $keys): boo
305308
protected function checkForInvalidAttributeValues(array $structure, array $keys): bool
306309
{
307310
foreach ($structure as $key => $value) {
311+
if (Operator::isOperator($value)) {
312+
// Set the attribute name on the operator for validation
313+
$value->setAttribute($key);
314+
315+
$operatorValidator = new OperatorValidator($this->collection, $this->currentDocument);
316+
if (!$operatorValidator->isValid($value)) {
317+
$this->message = $operatorValidator->getDescription();
318+
return false;
319+
}
320+
continue;
321+
}
322+
308323
$attribute = $keys[$key] ?? [];
309324
$type = $attribute['type'] ?? '';
310325
$array = $attribute['array'] ?? false;

0 commit comments

Comments
 (0)