Skip to content

Fix: Make firebase arrays work correctly #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abnegate merged 6 commits into from
Feb 13, 2025
Merged

Fix: Make firebase arrays work correctly #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
913cda7
Start work on fixing parts of firebase
PineappleIOnic Feb 6, 2025
225c273
Run Linter
PineappleIOnic Feb 12, 2025
4eb7a78
Fix type casting
PineappleIOnic Feb 13, 2025
0fc6300
Add PHPStan
PineappleIOnic Feb 13, 2025
8ccade2
Merge branch '0.6.x' into feat-fix-firebase-arrays
PineappleIOnic Feb 13, 2025
35788f8
Make PHPStan level 3
PineappleIOnic Feb 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .github/workflows/codeql-analysis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
name: "CodeQL"

on: [ pull_request ]
jobs:
lint:
name: CodeQL
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 2

- run: git checkout HEAD^2

- name: Run CodeQL
run: |
docker run --rm -v $PWD:/app composer sh -c \
"composer install --profile --ignore-platform-reqs && composer check"
2 changes: 1 addition & 1 deletion composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"test": "./vendor/bin/phpunit",
"lint": "./vendor/bin/pint --test",
"format": "./vendor/bin/pint",
"check": "./vendor/bin/phpstan analyse --level=8 --memory-limit=2G src tests"
"check": "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G"
},
"require": {
"php": ">=8.1",
Expand Down
14 changes: 7 additions & 7 deletions composer.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion src/Migration/Destinations/Appwrite.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -943,7 +943,7 @@ protected function createDocument(Document $resource, bool $isLast): bool
continue;
}

/** @var $attribute \Utopia\Database\Document */
/** @var \Utopia\Database\Document $attribute */
$found = false;
foreach ($collection->getAttribute('attributes', []) as $attribute) {
if ($attribute->getAttribute('key') == $key) {
Expand Down
2 changes: 1 addition & 1 deletion src/Migration/Resource.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ public function setMessage(string $message): self
}

/**
* @returns array<string>
* @return array<string>
*/
public function getPermissions(): array
{
Expand Down
5 changes: 3 additions & 2 deletions src/Migration/Sources/Appwrite.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -643,7 +643,7 @@ private function exportDocuments(int $batchSize): void

$attributes = $this->cache->get(Attribute::getName());
foreach ($attributes as $attribute) {
/** @var Attribute|Relationship $attribute */
/** @var Relationship $attribute */
if (
$attribute->getCollection()->getId() === $collection->getId() &&
$attribute->getType() === Attribute::TYPE_RELATIONSHIP &&
Expand All @@ -660,6 +660,7 @@ private function exportDocuments(int $batchSize): void
$manyToMany[] = $attribute->getKey();
}
}
/** @var Attribute|Relationship $attribute */

$queries[] = Query::select($selects);

Expand Down Expand Up @@ -1132,7 +1133,7 @@ protected function exportGroupStorage(int $batchSize, array $resources): void

try {
if (in_array(Resource::TYPE_BUCKET, $resources)) {
$this->exportBuckets($batchSize, true);
$this->exportBuckets($batchSize);
}
} catch (\Throwable $e) {
$this->addError(
Expand Down
64 changes: 39 additions & 25 deletions src/Migration/Sources/Firebase.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ protected function call(string $method, string $path = '', array $headers = [],
return parent::call($method, $path, $headers, $params, $responseHeaders);
}

/**
* @return array<string>
*/
public static function getSupportedResources(): array
{
return [
Expand Down Expand Up @@ -272,10 +275,11 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void
throw $e;
}

$database = new Database('default', 'default');
$database->setOriginalId('(default)');

try {
if (\in_array(Resource::TYPE_DATABASE, $resources)) {
$database = new Database('default', 'default');
$database->setOriginalId('(default)');
$this->callback([$database]);
}
} catch (\Throwable $e) {
Expand Down Expand Up @@ -368,23 +372,23 @@ private function exportDB(int $batchSize, bool $pushDocuments, Database $databas
/**
* @throws \Exception
*/
private function convertAttribute(Collection $collection, string $key, array $field): Attribute
private function convertAttribute(Collection $collection, string $key, array $field, bool $array = false): Attribute
{
if (array_key_exists('booleanValue', $field)) {
return new Boolean(
$key,
$collection,
required:false,
default: null,
array: false,
array: $array,
);
} elseif (array_key_exists('bytesValue', $field)) {
return new Text(
$key,
$collection,
required: false,
default: null,
array: false,
array: $array,
size: 1000000,
);
} elseif (array_key_exists('doubleValue', $field)) {
Expand All @@ -393,23 +397,23 @@ private function convertAttribute(Collection $collection, string $key, array $fi
$collection,
required: false,
default: null,
array: false,
array: $array,
);
} elseif (array_key_exists('integerValue', $field)) {
return new Integer(
$key,
$collection,
required: false,
default: null,
array: false,
array: $array,
);
} elseif (array_key_exists('mapValue', $field)) {
return new Text(
$key,
$collection,
required: false,
default: null,
array: false,
array: $array,
size: 1000000,
);
} elseif (array_key_exists('nullValue', $field)) {
Expand All @@ -418,7 +422,7 @@ private function convertAttribute(Collection $collection, string $key, array $fi
$collection,
required: false,
default: null,
array: false,
array: $array,
size: 1000000,
);
} elseif (array_key_exists('referenceValue', $field)) {
Expand All @@ -427,7 +431,7 @@ private function convertAttribute(Collection $collection, string $key, array $fi
$collection,
required: false,
default: null,
array: false,
array: $array,
size: 1000000,
); //TODO: This should be a reference attribute
} elseif (array_key_exists('stringValue', $field)) {
Expand All @@ -436,7 +440,7 @@ private function convertAttribute(Collection $collection, string $key, array $fi
$collection,
required: false,
default: null,
array: false,
array: $array,
size: 1000000,
);
} elseif (array_key_exists('timestampValue', $field)) {
Expand All @@ -445,15 +449,15 @@ private function convertAttribute(Collection $collection, string $key, array $fi
$collection,
required: false,
default: null,
array: false,
array: $array,
);
} elseif (array_key_exists('geoPointValue', $field)) {
return new Text(
$key,
$collection,
required: false,
default: null,
array: false,
array: $array,
size: 1000000,
);
} elseif (array_key_exists('arrayValue', $field)) {
Expand All @@ -470,19 +474,17 @@ private function calculateArrayType(Collection $collection, string $key, array $

foreach ($data['values'] as $field) {
if (! $previousType) {
$previousType = $this->convertAttribute($collection, $key, $field);
$previousType = $this->convertAttribute($collection, $key, $field, true);
} elseif ($previousType->getName() != ($this->convertAttribute($collection, $key, $field))->getName()) {
$isSameType = false;
break;
}
}

if ($isSameType) {
$previousType->setArray(true);

return $previousType;
} else {
return new Text($key, $collection, false, true, null, 1000000);
return new Text($key, $collection, false, true, true, 1000000);
}
}

Expand Down Expand Up @@ -521,7 +523,7 @@ private function exportCollection(Collection $collection, int $batchSize, bool $
}
}

$documents[] = $this->convertDocument($collection, $document);
$documents[] = $this->convertDocument($collection, $document, $documentSchema);
}

// Transfer Documents
Expand Down Expand Up @@ -557,13 +559,13 @@ private function exportCollection(Collection $collection, int $batchSize, bool $
private function calculateValue(array $field)
{
if (array_key_exists('booleanValue', $field)) {
return $field['booleanValue'];
return boolval($field['booleanValue']);
} elseif (array_key_exists('bytesValue', $field)) {
return $field['bytesValue'];
} elseif (array_key_exists('doubleValue', $field)) {
return $field['doubleValue'];
return floatval($field['doubleValue']);
} elseif (array_key_exists('integerValue', $field)) {
return $field['integerValue'];
return intval($field['integerValue']);
} elseif (array_key_exists('mapValue', $field)) {
return json_encode($field['mapValue']);
} elseif (array_key_exists('nullValue', $field)) {
Expand All @@ -575,21 +577,33 @@ private function calculateValue(array $field)
} elseif (array_key_exists('timestampValue', $field)) {
return $field['timestampValue'];
} elseif (array_key_exists('geoPointValue', $field)) {
return [$field['geoPointValue']['latitude'], $field['geoPointValue']['longitude']];
return json_encode($field['geoPointValue']);
} elseif (array_key_exists('arrayValue', $field)) {
//TODO:
$values = [];
foreach ($field['arrayValue']['values'] as $value) {
$values[] = $this->calculateValue($value);
}
return array_values($values);
} elseif (array_key_exists('referenceValue', $field)) {
//TODO:
} else {
throw new \Exception('Unknown field type');
}
}

private function convertDocument(Collection $collection, array $document): Document
private function convertDocument(Collection $collection, array $document, array $documentSchema): Document
{
$data = [];
foreach ($document['fields'] as $key => $field) {
$data[$key] = $this->calculateValue($field);
$value = $this->calculateValue($field);

if ($documentSchema[$key]->getType() === Attribute::TYPE_STRING && is_array($value)) {
$value = array_map(function ($item) {
return strval($item);
}, $value);
};

$data[$key] = $value;
}

$documentId = explode('/', $document['name']);
Expand Down
8 changes: 4 additions & 4 deletions src/Migration/Transfer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,6 @@ public function __construct(Source $source, Destination $destination)
$this->source->registerCache($this->cache);
$this->destination->registerCache($this->cache);
$this->destination->setSource($source);

return $this;
}

public function getStatusCounters(): array
Expand Down Expand Up @@ -186,8 +184,8 @@ public function getStatusCounters(): array
public function run(
array $resources,
callable $callback,
string $rootResourceId = null,
string $rootResourceType = null,
?string $rootResourceId = null,
?string $rootResourceType = null,
): void {
// Allows you to push entire groups if you want.
$computedResources = [];
Expand All @@ -196,12 +194,14 @@ public function run(

foreach ($resources as $resource) {
if (is_array($resource)) {
/** @var array<string> $resource */
$computedResources = array_merge($computedResources, $resource);
} else {
$computedResources[] = $resource;
}
}

/** @var array<string> $computedResources */
$computedResources = array_map('strtolower', $computedResources);

if ($rootResourceId !== '') {
Expand Down