fix: do not use excessive memory while parsing packfiles (#5)

Author: ForbesLindesay

.dockerignore

@@ -0,0 +1 @@
+node_modules

.github/workflows/test.yml

@@ -24,3 +24,16 @@ jobs:
       - run: yarn install --frozen-lockfile
       - run: yarn build
       - run: yarn test
+
+  test-memory-usage:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 16.x
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - name: Check memory usage for packfile parser
+        run: node scripts/test-limited-memory-parse scripts/packfiles/nodejs-node.dat

.gitignore

@@ -96,4 +96,6 @@ bundle-cache
 package-entry-points.json
 
 secrets/
-packages/packfile/samples/output.pack
+packages/packfile/samples/output.pack
+
+temp/

Dockerfile

@@ -0,0 +1,7 @@
+FROM git-client-deps
+
+WORKDIR /app
+
+ADD . /app
+
+CMD node --max_old_space_size=128 scripts/test-parse

package.json

@@ -23,6 +23,7 @@
     "is-builtin-module": "^3.0.0",
     "jest": "^26.0.1",
     "lint-staged": "^10.1.3",
+    "parameter-reducers": "^2.0.0",
     "prettier": "^2.0.4",
     "react": "^17.0.1",
     "react-dom": "^17.0.1",

packages/http/README.md

@@ -51,7 +51,7 @@ After the initial request, all other requests require an object detailing th ser
 
 #### DEFAULT_HTTP_HANDLER
 
-An implementation of `HttpInterface` using `cross-fetch`. You can wrap these if you need to set additional headers (e.g. for auth).
+An implementation of `HttpInterface` using `http-basic`. You can wrap these if you need to set additional headers (e.g. for auth).
 
 ### Methods
 

packages/http/package.json

@@ -20,7 +20,6 @@
   },
   "dependencies": {
     "@rollingversions/git-protocol": "^0.0.0",
-    "cross-fetch": "^3.0.6",
     "http-basic": "^8.1.3",
     "https-proxy-agent": "^5.0.0"
   },

packages/http/src/fetchObjects.ts

@@ -3,6 +3,9 @@ import {
   parseFetchResponse,
   FetchCommand,
   FetchResponseEntryObject,
+  FetchCommandOutputOptions,
+  parseFetchResponseV2,
+  FetchCommandOutputOptionsV2,
 } from '@rollingversions/git-protocol';
 import {ContextWithServerCapabilities} from './Context';
 
@@ -16,7 +19,12 @@ export default async function fetchObjects<
 >(
   repoURL: URL,
   command: FetchCommand,
-  ctx: ContextWithServerCapabilities<THeaders>,
+  {
+    raw,
+    store,
+    storeMode,
+    ...ctx
+  }: ContextWithServerCapabilities<THeaders> & FetchCommandOutputOptions,
 ) {
   const url = new URL(
     `${
@@ -56,5 +64,64 @@ export default async function fetchObjects<
       )}`,
     );
   }
-  return parseFetchResponse(response.body);
+  return parseFetchResponse(response.body, {raw, store, storeMode});
+}
+
+export async function* fetchObjectsV2<
+  THeaders extends {set(name: string, value: string): unknown}
+>(
+  repoURL: URL,
+  command: FetchCommand,
+  {
+    onProgress,
+    store,
+    storeMode,
+    ...ctx
+  }: ContextWithServerCapabilities<THeaders> & FetchCommandOutputOptionsV2,
+) {
+  const url = new URL(
+    `${
+      repoURL.href.endsWith('.git') ? repoURL.href : `${repoURL.href}.git`
+    }/git-upload-pack`,
+  );
+  const headers = ctx.http.createHeaders(url);
+  headers.set('accept', 'application/x-git-upload-pack-result');
+  headers.set('content-type', 'application/x-git-upload-pack-request');
+  headers.set('git-protocol', 'version=2');
+  headers.set('user-agent', ctx.agent);
+
+  const response = await ctx.http.post(
+    url,
+    headers,
+    composeFetchCommand(
+      command,
+      new Map(
+        [
+          ['agent', ctx.agent] as const,
+          ...defaultCapabilities,
+        ].filter(([key]) => ctx.serverCapabilities.has(key)),
+      ),
+    ),
+  );
+  if (response.statusCode !== 200) {
+    const body = await new Promise<Buffer>((resolve, reject) => {
+      const body: Buffer[] = [];
+      response.body
+        .on(`data`, (chunk) => body.push(chunk))
+        .on(`error`, reject)
+        .on(`end`, () => resolve(Buffer.concat(body)));
+    });
+    throw new Error(
+      `Git server responded with status ${response.statusCode}: ${body.toString(
+        `utf8`,
+      )}`,
+    );
+  }
+  for await (const entry of parseFetchResponseV2(response.body, {
+    onProgress,
+    store,
+    storeMode,
+  })) {
+    yield entry;
+  }
 }

packages/packfile/package.json

@@ -19,9 +19,7 @@
     "build": "tsc"
   },
   "dependencies": {
-    "@rollingversions/git-core": "^0.0.0",
-    "@types/pako": "^1.0.2",
-    "pako": "^1.0.5"
+    "@rollingversions/git-core": "^0.0.0"
   },
   "devDependencies": {
     "@types/node": "*"