Skip to content

fix chunk upload #648

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lohanidamodar merged 4 commits into from
Jul 27, 2023
Merged

fix chunk upload #648

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
398feb9
fix chunk upload dart and flutter
lohanidamodar Apr 19, 2023
dee773a
fix dart and flutter chunk upload
lohanidamodar Apr 19, 2023
0add3b6
fix chunk upload for remaining SDKs
lohanidamodar Apr 19, 2023
64e6376
Merge branch '1.4.x' into fix-chunked-upload
lohanidamodar Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions templates/android/library/src/main/java/io/appwrite/Client.kt.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ class Client @JvmOverloads constructor(
responseType = Map::class.java,
)
val chunksUploaded = current["chunksUploaded"] as Long
offset = (chunksUploaded * CHUNK_SIZE).coerceAtMost(size)
offset = chunksUploaded * CHUNK_SIZE
}

while (offset < size) {
Expand All @@ -385,7 +385,7 @@ class Client @JvmOverloads constructor(
}
"bytes" -> {
val end = if (offset + CHUNK_SIZE < size) {
offset + CHUNK_SIZE
offset + CHUNK_SIZE - 1
} else {
size - 1
}
Expand All @@ -405,7 +405,7 @@ class Client @JvmOverloads constructor(
)

headers["Content-Range"] =
"bytes $offset-${((offset + CHUNK_SIZE) - 1).coerceAtMost(size)}/$size"
"bytes $offset-${((offset + CHUNK_SIZE) - 1).coerceAtMost(size - 1)}/$size"

result = call(
method = "POST",
Expand Down
2 changes: 1 addition & 1 deletion templates/cli/lib/commands/command.js.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ const {{ service.name | caseLower }}{{ method.name | caseUcfirst }} = async ({ {

for (counter; counter < totalCounters; counter++) {
const start = (counter * libClient.CHUNK_SIZE);
const end = Math.min((((counter * libClient.CHUNK_SIZE) + libClient.CHUNK_SIZE) - 1), size);
const end = Math.min((((counter * libClient.CHUNK_SIZE) + libClient.CHUNK_SIZE) - 1), size - 1);

headers['content-range'] = 'bytes ' + start + '-' + end + '/' + size;

Expand Down
12 changes: 6 additions & 6 deletions templates/dart/lib/src/client_browser.dart.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,18 +114,18 @@ class ClientBrowser extends ClientBase with ClientMixin {
headers: headers,
);
final int chunksUploaded = res.data['chunksUploaded'] as int;
offset = min(size, chunksUploaded * CHUNK_SIZE);
offset = chunksUploaded * CHUNK_SIZE;
} on {{spec.title | caseUcfirst}}Exception catch (_) {}
}

while (offset < size) {
List<int> chunk;
final end = min(offset + CHUNK_SIZE, size);
var chunk;
final end = min(offset + CHUNK_SIZE - 1, size - 1);
chunk = file.bytes!.getRange(offset, end).toList();
params[paramName] =
http.MultipartFile.fromBytes(paramName, chunk, filename: file.filename);
headers['content-range'] =
'bytes $offset-${min<int>(((offset + CHUNK_SIZE) - 1), size)}/$size';
'bytes $offset-${min<int>((offset + CHUNK_SIZE - 1), size - 1)}/$size';
res = await call(HttpMethod.post,
path: path, headers: headers, params: params);
offset += CHUNK_SIZE;
Expand All @@ -134,8 +134,8 @@ class ClientBrowser extends ClientBase with ClientMixin {
}
final progress = UploadProgress(
$id: res.data['\$id'] ?? '',
progress: min(offset - 1, size) / size * 100,
sizeUploaded: min(offset - 1, size),
progress: min(offset, size) / size * 100,
sizeUploaded: min(offset, size),
chunksTotal: res.data['chunksTotal'] ?? 0,
chunksUploaded: res.data['chunksUploaded'] ?? 0,
);
Expand Down
10 changes: 5 additions & 5 deletions templates/dart/lib/src/client_io.dart.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ class ClientIO extends ClientBase with ClientMixin {
headers: headers,
);
final int chunksUploaded = res.data['chunksUploaded'] as int;
offset = min(size, chunksUploaded * CHUNK_SIZE);
offset = chunksUploaded * CHUNK_SIZE;
} on {{spec.title | caseUcfirst}}Exception catch (_) {}
}

Expand All @@ -156,7 +156,7 @@ class ClientIO extends ClientBase with ClientMixin {
while (offset < size) {
List<int> chunk = [];
if (file.bytes != null) {
final end = min(offset + CHUNK_SIZE-1, size-1);
final end = min(offset + CHUNK_SIZE - 1, size - 1);
chunk = file.bytes!.getRange(offset, end).toList();
} else {
raf!.setPositionSync(offset);
Expand All @@ -165,7 +165,7 @@ class ClientIO extends ClientBase with ClientMixin {
params[paramName] =
http.MultipartFile.fromBytes(paramName, chunk, filename: file.filename);
headers['content-range'] =
'bytes $offset-${min<int>(((offset + CHUNK_SIZE) - 1), size)}/$size';
'bytes $offset-${min<int>((offset + CHUNK_SIZE - 1), size - 1)}/$size';
res = await call(HttpMethod.post,
path: path, headers: headers, params: params);
offset += CHUNK_SIZE;
Expand All @@ -174,8 +174,8 @@ class ClientIO extends ClientBase with ClientMixin {
}
final progress = UploadProgress(
$id: res.data['\$id'] ?? '',
progress: min(offset - 1, size) / size * 100,
sizeUploaded: min(offset - 1, size),
progress: min(offset, size) / size * 100,
sizeUploaded: min(offset, size),
chunksTotal: res.data['chunksTotal'] ?? 0,
chunksUploaded: res.data['chunksUploaded'] ?? 0,
);
Expand Down
10 changes: 5 additions & 5 deletions templates/flutter/lib/src/client_browser.dart.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,18 +143,18 @@ class ClientBrowser extends ClientBase with ClientMixin {
headers: headers,
);
final int chunksUploaded = res.data['chunksUploaded'] as int;
offset = min(size, chunksUploaded * CHUNK_SIZE);
offset = chunksUploaded * CHUNK_SIZE;
} on {{spec.title | caseUcfirst}}Exception catch (_) {}
}

while (offset < size) {
var chunk;
final end = min(offset + CHUNK_SIZE, size);
final end = min(offset + CHUNK_SIZE - 1, size - 1);
chunk = file.bytes!.getRange(offset, end).toList();
params[paramName] =
http.MultipartFile.fromBytes(paramName, chunk, filename: file.filename);
headers['content-range'] =
'bytes $offset-${min<int>(((offset + CHUNK_SIZE) - 1), size)}/$size';
'bytes $offset-${min<int>((offset + CHUNK_SIZE - 1), size - 1)}/$size';
res = await call(HttpMethod.post,
path: path, headers: headers, params: params);
offset += CHUNK_SIZE;
Expand All @@ -163,8 +163,8 @@ class ClientBrowser extends ClientBase with ClientMixin {
}
final progress = UploadProgress(
$id: res.data['\$id'] ?? '',
progress: min(offset - 1, size) / size * 100,
sizeUploaded: min(offset - 1, size),
progress: min(offset, size) / size * 100,
sizeUploaded: min(offset, size),
chunksTotal: res.data['chunksTotal'] ?? 0,
chunksUploaded: res.data['chunksUploaded'] ?? 0,
);
Expand Down
10 changes: 5 additions & 5 deletions templates/flutter/lib/src/client_io.dart.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ class ClientIO extends ClientBase with ClientMixin {
headers: headers,
);
final int chunksUploaded = res.data['chunksUploaded'] as int;
offset = min(size, chunksUploaded * CHUNK_SIZE);
offset = chunksUploaded * CHUNK_SIZE;
} on {{spec.title | caseUcfirst}}Exception catch (_) {}
}

Expand All @@ -276,7 +276,7 @@ class ClientIO extends ClientBase with ClientMixin {
while (offset < size) {
List<int> chunk = [];
if (file.bytes != null) {
final end = min(offset + CHUNK_SIZE-1, size-1);
final end = min(offset + CHUNK_SIZE - 1, size - 1);
chunk = file.bytes!.getRange(offset, end).toList();
} else {
raf!.setPositionSync(offset);
Expand All @@ -285,7 +285,7 @@ class ClientIO extends ClientBase with ClientMixin {
params[paramName] =
http.MultipartFile.fromBytes(paramName, chunk, filename: file.filename);
headers['content-range'] =
'bytes $offset-${min<int>(((offset + CHUNK_SIZE) - 1), size)}/$size';
'bytes $offset-${min<int>((offset + CHUNK_SIZE - 1), size - 1)}/$size';
res = await call(HttpMethod.post,
path: path, headers: headers, params: params);
offset += CHUNK_SIZE;
Expand All @@ -294,8 +294,8 @@ class ClientIO extends ClientBase with ClientMixin {
}
final progress = UploadProgress(
$id: res.data['\$id'] ?? '',
progress: min(offset - 1, size) / size * 100,
sizeUploaded: min(offset - 1, size),
progress: min(offset, size) / size * 100,
sizeUploaded: min(offset, size),
chunksTotal: res.data['chunksTotal'] ?? 0,
chunksUploaded: res.data['chunksUploaded'] ?? 0,
);
Expand Down
6 changes: 3 additions & 3 deletions templates/kotlin/src/main/kotlin/io/appwrite/Client.kt.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ class Client @JvmOverloads constructor(
responseType = Map::class.java,
)
val chunksUploaded = current["chunksUploaded"] as Long
offset = (chunksUploaded * CHUNK_SIZE).coerceAtMost(size)
offset = chunksUploaded * CHUNK_SIZE
}

while (offset < size) {
Expand All @@ -344,7 +344,7 @@ class Client @JvmOverloads constructor(
}
"bytes" -> {
val end = if (offset + CHUNK_SIZE < size) {
offset + CHUNK_SIZE
offset + CHUNK_SIZE - 1
} else {
size - 1
}
Expand All @@ -364,7 +364,7 @@ class Client @JvmOverloads constructor(
)

headers["Content-Range"] =
"bytes $offset-${((offset + CHUNK_SIZE) - 1).coerceAtMost(size)}/$size"
"bytes $offset-${((offset + CHUNK_SIZE) - 1).coerceAtMost(size - 1)}/$size"

result = call(
method = "POST",
Expand Down
2 changes: 1 addition & 1 deletion templates/node/base/requests/file.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
}

const start = currentChunkStart;
const end = Math.min(((start + client.CHUNK_SIZE) - 1), size);
const end = Math.min(((start + client.CHUNK_SIZE) - 1), size - 1);

if(!lastUpload || currentChunkStart !== 0) {
headers['content-range'] = 'bytes ' + start + '-' + end + '/' + size;
Expand Down
4 changes: 2 additions & 2 deletions templates/php/base/requests/file.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
$chunk = substr($file->getData(), $start, Client::CHUNK_SIZE);
}
$params['{{ parameter.name }}'] = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode($chunk), $mimeType, $postedName);
$headers['content-range'] = 'bytes ' . ($counter * Client::CHUNK_SIZE) . '-' . min(((($counter * Client::CHUNK_SIZE) + Client::CHUNK_SIZE) - 1), $size) . '/' . $size;
$headers['content-range'] = 'bytes ' . ($counter * Client::CHUNK_SIZE) . '-' . min(((($counter * Client::CHUNK_SIZE) + Client::CHUNK_SIZE) - 1), $size - 1) . '/' . $size;
if(!empty($id)) {
$headers['x-{{spec.title | caseLower }}-id'] = $id;
}
Expand All @@ -81,7 +81,7 @@
if($onProgress !== null) {
$onProgress([
'$id' => $response['$id'],
'progress' => min(((($counter * Client::CHUNK_SIZE) + Client::CHUNK_SIZE) - 1), $size) / $size * 100,
'progress' => min(((($counter * Client::CHUNK_SIZE) + Client::CHUNK_SIZE)), $size) / $size * 100,
'sizeUploaded' => min($counter * Client::CHUNK_SIZE),
'chunksTotal' => $response['chunksTotal'],
'chunksUploaded' => $response['chunksUploaded'],
Expand Down
4 changes: 2 additions & 2 deletions templates/python/package/client.py.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ class Client:
input_file.data = input[offset:end]

params[param_name] = input_file
headers["content-range"] = f'bytes {offset}-{min((offset + self._chunk_size) - 1, size)}/{size}'
headers["content-range"] = f'bytes {offset}-{min((offset + self._chunk_size) - 1, size - 1)}/{size}'

result = self.call(
'post',
Expand All @@ -175,7 +175,7 @@ class Client:
headers["x-{{ spec.title | caseLower }}-id"] = result["$id"]

if on_progress is not None:
end = min((((counter * self._chunk_size) + self._chunk_size) - 1), size)
end = min((((counter * self._chunk_size) + self._chunk_size) - 1), size - 1)
on_progress({
"$id": result["$id"],
"progress": min(offset, size)/size * 100,
Expand Down
4 changes: 2 additions & 2 deletions templates/ruby/lib/container/client.rb.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ module {{ spec.title | caseUcfirst }}
params: {}
)
chunks_uploaded = current['chunksUploaded'].to_i
offset = [size, (chunks_uploaded * @chunk_size)].min
offset = chunks_uploaded * @chunk_size
end

while offset < size
Expand All @@ -158,7 +158,7 @@ module {{ spec.title | caseUcfirst }}
mime_type: input_file.mime_type
)

headers['content-range'] = "bytes #{offset}-#{[offset + @chunk_size - 1, size].min}/#{size}"
headers['content-range'] = "bytes #{offset}-#{[offset + @chunk_size - 1, size - 1].min}/#{size}"

result = call(
method: 'POST',
Expand Down
4 changes: 2 additions & 2 deletions templates/swift/Sources/Client.swift.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ open class Client {
converter: { return $0 as! [String: Any] }
)
let chunksUploaded = map["chunksUploaded"] as! Int
offset = min(size, (chunksUploaded * Client.chunkSize))
offset = chunksUploaded * Client.chunkSize
} catch {
// File does not exist yet, swallow exception
}
Expand All @@ -390,7 +390,7 @@ open class Client {
?? (input.data as! ByteBuffer).getSlice(at: offset, length: Int(size - offset))

params[paramName] = InputFile.fromBuffer(slice!, filename: input.filename, mimeType: input.mimeType)
headers["content-range"] = "bytes \(offset)-\(min((offset + Client.chunkSize) - 1, size))/\(size)"
headers["content-range"] = "bytes \(offset)-\(min((offset + Client.chunkSize) - 1, size - 1))/\(size)"

result = try await call(
method: "POST",
Expand Down
2 changes: 1 addition & 1 deletion templates/web/src/services/template.ts.twig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ export class {{ service.name | caseUcfirst }} extends Service {
if (onProgress) {
onProgress({
$id: response.$id,
progress: Math.min((counter + 1) * Service.CHUNK_SIZE - 1, size) / size * 100,
progress: Math.min((counter + 1) * Service.CHUNK_SIZE, size) / size * 100,
sizeUploaded: end,
chunksTotal: response.chunksTotal,
chunksUploaded: response.chunksUploaded
Expand Down