Skip to content

Improve Firebase Stability #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eldadfux merged 4 commits into from
Aug 16, 2023
Merged

Improve Firebase Stability #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d3aaa7e
Add isset checks for 0 data received on certain endpoints
PineappleIOnic Aug 10, 2023
0b39e25
Update Firebase.php
PineappleIOnic Aug 11, 2023
957b7cb
Update Firebase.php
PineappleIOnic Aug 11, 2023
82660c4
Add checks for if services are disabled
PineappleIOnic Aug 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions playground.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,25 +69,23 @@
* Initialise Transfer Class
*/
$transfer = new Transfer(
$sourceSupabase,
$sourceFirebase,
$destinationLocal
);

$sourceFirebase->report();

// $sourceSupabase->report();

// /**
// * Run Transfer
// */
// $transfer->run($sourceAppwrite->getSupportedResources(),
// function (array $resources) {
// }
// );
$transfer->run($sourceFirebase->getSupportedResources(),
function (array $resources) {
}
);

// $report = [];
$report = [];

// $cache = $transfer->getCache()->getAll();
$cache = $transfer->getCache()->getAll();

// foreach ($cache as $type => $resources) {
// foreach ($resources as $resource) {
Expand Down
83 changes: 77 additions & 6 deletions src/Migration/Sources/Firebase.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,20 @@ public function report(array $resources = []): array

protected function exportGroupAuth(int $batchSize, array $resources)
{
// Check if Auth is enabled
try {
$this->call('GET', 'https://identitytoolkit.googleapis.com/v1/projects');
} catch (\Exception $e) {
$message = json_decode($e->getMessage(), true);

if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') {
// IdentityKit is disabled
return;
}

throw $e;
}

if (in_array(Resource::TYPE_USER, $resources)) {
$this->exportUsers($batchSize);
}
Expand Down Expand Up @@ -181,6 +195,10 @@ private function exportUsers(int $batchSize)
'Content-Type' => 'application/json',
], $request);

if (! isset($response['users'])) {
break;
}

$result = $response['users'];
$nextPageToken = $response['nextPageToken'] ?? null;

Expand Down Expand Up @@ -234,6 +252,20 @@ private function calculateUserType(array $providerData): array

protected function exportGroupDatabases(int $batchSize, array $resources)
{
// Check if Firestore is enabled
try {
$this->call('GET', 'https://firestore.googleapis.com/v1/projects/'.$this->projectID.'/databases');
} catch (\Exception $e) {
$message = json_decode($e->getMessage(), true);

if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') {
// Firestore is disabled
return;
}

throw $e;
}

if (in_array(Resource::TYPE_DATABASE, $resources)) {
$database = new Database('default', 'default');
$database->setOriginalId('(default)');
Expand All @@ -254,12 +286,29 @@ private function exportDB(int $batchSize, bool $pushDocuments, Database $databas
while (true) {
$collections = [];

$result = $this->call('POST', $baseURL.':listCollectionIds', [
'Content-Type' => 'application/json',
], [
'pageSize' => $batchSize,
'pageToken' => $nextPageToken,
]);
try {
$result = $this->call('POST', $baseURL.':listCollectionIds', [
'Content-Type' => 'application/json',
], [
'pageSize' => $batchSize,
'pageToken' => $nextPageToken,
]);

if (! isset($result['collectionIds'])) {
break;
}
} catch (\Exception $e) {
if ($e->getCode() == 403) {
$errorMessage = new Collection($database, 'firestore', 'firestore');

$errorMessage->setStatus(Resource::STATUS_ERROR);
$errorMessage->setMessage($e->getMessage());

$this->cache->add($errorMessage);
}

break;
}

// Transfer Collections
foreach ($result['collectionIds'] as $collection) {
Expand Down Expand Up @@ -437,6 +486,24 @@ private function convertDocument(Collection $collection, array $document): Docum

protected function exportGroupStorage(int $batchSize, array $resources)
{
// Check if storage is enabled
try {
$this->call('GET', 'https://storage.googleapis.com/storage/v1/b', [], [
'project' => $this->projectID,
'maxResults' => 1,
'alt' => 'json',
]);
} catch (\Exception $e) {
$message = json_decode($e->getMessage(), true);

if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') {
// Storage is disabled
return;
}

throw $e;
}

if (in_array(Resource::TYPE_BUCKET, $resources)) {
$this->exportBuckets($batchSize);
}
Expand Down Expand Up @@ -464,6 +531,10 @@ private function exportBuckets(int $batchsize)
break;
}

if (! isset($result['items'])) {
break;
}

foreach ($result['items'] as $bucket) {
$this->callback([new Bucket($bucket['id'], $bucket['name'], [], false)]);
}
Expand Down
17 changes: 8 additions & 9 deletions src/Migration/Transfer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,19 +88,18 @@ public function getStatusCounters()
];
}

if ($this->source->previousReport) {
foreach ($this->source->previousReport as $resource => $data) {
if ($resource != 'size' && $resource != 'version') {
$status[$resource]['pending'] = $data;
}
}
}

foreach ($this->cache->getAll() as $resources) {
foreach ($resources as $resource) {
/** @var resource $resource */
$status[$resource->getName()][$resource->getStatus()]++;
$status[$resource->getName()]['pending']--;
}
}

if ($this->source->previousReport) {
foreach ($this->source->previousReport as $resource => $data) {
if ($resource != 'size' && $resource != 'version' && isset($status[$resource])) {
$status[$resource]['pending'] = $data;
}
}
}

Expand Down