feat(project): GitHub reader pins a commit, caches, fails over and enforces limits (G-A2)

apps/datatug-app/e2e/github-store.spec.ts

@@ -82,11 +82,116 @@ function getQueryBodyText(page: Page): Promise<string | undefined> {
   });
 }
 
+/** Every request the page makes to the three hosts a GitHub project is read from. */
+function watchGithubRequests(page: Page): string[] {
+  const urls: string[] = [];
+  page.on('request', (request) => {
+    const { hostname } = new URL(request.url());
+    if (
+      ['api.github.com', 'raw.githubusercontent.com', 'cdn.jsdelivr.net'].includes(hostname)
+    ) {
+      urls.push(request.url());
+    }
+  });
+  return urls;
+}
+
+/**
+ * G-A2 (design `demo-as-github-project.md` 4.5): the requests of a cold visit and a warm one, against the real
+ * GitHub. Anonymous `api.github.com` access is 60 requests per hour per address, so the budget IS the acceptance:
+ * a cold visit resolves the commit (1 call) and lists the repo (1 call); every file is read at that one commit;
+ * nothing else is asked of GitHub, and no mirror is used. A warm visit (same browser, within 5 minutes) asks
+ * nothing of GitHub at all.
+ */
+test.describe('GitHub-store project — request budget (G-A2)', () => {
+  const SHA_IN_URL = /^https:\/\/raw\.githubusercontent\.com\/[^/]+\/[^/]+\/([0-9a-f]{40})\//;
+
+  async function cachedFileCount(page: Page): Promise<number> {
+    return page.evaluate(
+      () =>
+        new Promise<number>((resolve) => {
+          const open = indexedDB.open('datatug-github-files');
+          open.onerror = () => resolve(0);
+          open.onsuccess = () => {
+            const db = open.result;
+            if (!db.objectStoreNames.contains('files')) {
+              db.close();
+              resolve(0);
+              return;
+            }
+            const count = db.transaction('files').objectStore('files').count();
+            count.onsuccess = () => {
+              db.close();
+              resolve(count.result);
+            };
+            count.onerror = () => {
+              db.close();
+              resolve(0);
+            };
+          };
+        }),
+    );
+  }
+
+  test('a cold visit: 2 calls to the API and every file at one commit; a warm visit: nothing', async ({
+    page,
+  }) => {
+    const github = watchGithubRequests(page);
+    const errors = installErrorLoggerWatch(page);
+
+    await page.goto(PROJECT_URL);
+    await expect(
+      activePage(page).getByRole('tab', { name: 'Boards' }),
+    ).toBeVisible({ timeout: 20_000 });
+    await page.locator('sneat-datatug-project-menu-top ion-item', { hasText: 'Entities' }).click();
+    await expect(activePage(page).getByText('Album', { exact: true })).toBeVisible({
+      timeout: 15_000,
+    });
+
+    const api = github.filter((u) => new URL(u).hostname === 'api.github.com');
+    const raw = github.filter((u) => new URL(u).hostname === 'raw.githubusercontent.com');
+    console.log(`cold: api.github.com=${api.length} raw=${raw.length} requests=${github.length}`);
+    expect(api.length).toBeLessThanOrEqual(2);
+    expect(api.some((u) => u.includes('/commits/HEAD'))).toBe(true);
+    expect(api.filter((u) => u.includes('/git/trees/'))).toHaveLength(1);
+    expect(github.some((u) => u.includes('cdn.jsdelivr.net'))).toBe(false);
+    expect(raw.length).toBeGreaterThan(0);
+    const commits = new Set(raw.map((u) => SHA_IN_URL.exec(u)?.[1]));
+    expect(commits.size).toBe(1);
+    expect(commits.has(undefined)).toBe(false);
+    // The listing was read at that very commit, never at a branch.
+    expect(api.find((u) => u.includes('/git/trees/'))).toContain(`/git/trees/${[...commits][0]}?`);
+    // Each file at most once.
+    expect(new Set(raw).size).toBe(raw.length);
+
+    // Everything read is in the cache (the files and the listing) before the page is loaded again.
+    await expect
+      .poll(() => cachedFileCount(page), { timeout: 10_000 })
+      .toBeGreaterThanOrEqual(raw.length + 1);
+
+    // Warm: the same browser, a new page load, within the 5 minutes the resolved commit is remembered.
+    github.length = 0;
+    await page.goto(PROJECT_URL);
+    await expect(
+      activePage(page).getByRole('tab', { name: 'Boards' }),
+    ).toBeVisible({ timeout: 20_000 });
+    await page.locator('sneat-datatug-project-menu-top ion-item', { hasText: 'Entities' }).click();
+    await expect(activePage(page).getByText('Album', { exact: true })).toBeVisible({
+      timeout: 15_000,
+    });
+    console.log(`warm: requests=${github.length}`);
+    expect(github).toEqual([]);
+
+    expect(errors).toEqual([]);
+  });
+});
+
 test.describe('GitHub-store project — every side-menu page loads without error', () => {
   test('project page, environments, entities, queries, boards, servers all render their real content', async ({
     page,
   }) => {
     const errors = installErrorLoggerWatch(page);
+    const github = watchGithubRequests(page);
 
     // --- Project page ---------------------------------------------------
     await page.goto(PROJECT_URL);
@@ -183,6 +288,17 @@ test.describe('GitHub-store project — every side-menu page loads without error
       });
     }
 
+    // The whole walk of the side menu, read at ONE commit: 2 calls to the API (the commit, the listing), every
+    // file once, no mirror (design 4.5).
+    const raw = github.filter((u) => u.startsWith('https://raw.githubusercontent.com/'));
+    console.log(
+      `full walk: api.github.com=${github.filter((u) => u.startsWith('https://api.github.com/')).length} raw=${raw.length}`,
+    );
+    expect(github.filter((u) => u.startsWith('https://api.github.com/'))).toHaveLength(2);
+    expect(github.filter((u) => u.startsWith('https://cdn.jsdelivr.net/'))).toEqual([]);
+    expect(new Set(raw).size).toBe(raw.length);
+    expect(new Set(raw.map((u) => u.split('/')[5])).size).toBe(1);
+
     expect(errors).toEqual([]);
   });
 

libs/datatug/main/src/lib/folders/core/datatug-folders.service.spec.ts

@@ -1,10 +1,18 @@
 import { HttpClient } from '@angular/common/http';
 import { TestBed } from '@angular/core/testing';
 import { Firestore } from 'firebase/firestore';
-import { of } from 'rxjs';
 
 import { DatatugFoldersService } from './datatug-folders.service';
 import { DatatugStoreServiceFactory } from '../../services/repo/datatug-store-service-factory.service';
+import {
+  FakeGithub,
+  fakeSha,
+} from '../../services/repo/github/github-fake-backend.test';
+import { NO_GITHUB_FILE_STORE } from '../../services/repo/github/github-file-store-api';
+import {
+  GITHUB_FETCH,
+  GITHUB_FILE_STORE,
+} from '../../services/repo/github/github-project-reader.service';
 
 describe('DatatugFoldersService', () => {
   describe('with a stubbed store service factory', () => {
@@ -102,36 +110,51 @@ describe('DatatugFoldersService', () => {
    */
   describe('with the real store service factory, for a GitHub-store project', () => {
     let service: DatatugFoldersService;
-    let httpGet: ReturnType<typeof vi.fn>;
+    let github: FakeGithub;
 
     beforeEach(() => {
-      httpGet = vi.fn(() =>
-        of({
+      // The project file is read through the GitHub reader (one commit, one cache), over a fake of GitHub.
+      github = new FakeGithub();
+      github.addRepo('datatug/datatug-demo-projects', fakeSha(1), {
+        'demo-project-1/datatug-project.json': JSON.stringify({
           id: 'datatug-demo-projects@datatug@demo-project-1',
           title: 'DataTug Demo Project 1',
           boards: [{ id: 'board1', title: '1st board' }],
         }),
-      );
+      });
       TestBed.configureTestingModule({
         providers: [
-          { provide: HttpClient, useValue: { get: httpGet } },
+          { provide: GITHUB_FETCH, useValue: github.fetch },
+          { provide: GITHUB_FILE_STORE, useValue: NO_GITHUB_FILE_STORE },
           { provide: Firestore, useValue: {} },
         ],
       });
       service = TestBed.inject(DatatugFoldersService);
     });
 
-    it('emits the real root folder once, with no error, for a GitHub-store project', () => {
+    it('emits the real root folder once, with no error, for a GitHub-store project', async () => {
       const next = vi.fn();
       const error = vi.fn();
       const complete = vi.fn();
-      service
-        .watchFolder({
-          storeId: 'github.com',
-          projectId: 'datatug-demo-projects@datatug@demo-project-1',
-          id: '~',
-        })
-        .subscribe({ next, error, complete });
+      await new Promise<void>((resolve) =>
+        service
+          .watchFolder({
+            storeId: 'github.com',
+            projectId: 'datatug-demo-projects@datatug@demo-project-1',
+            id: '~',
+          })
+          .subscribe({
+            next,
+            error: (e) => {
+              error(e);
+              resolve();
+            },
+            complete: () => {
+              complete();
+              resolve();
+            },
+          }),
+      );
 
       expect(error).not.toHaveBeenCalled();
       expect(complete).toHaveBeenCalledTimes(1);
@@ -162,7 +185,7 @@ describe('DatatugFoldersService', () => {
       // Never even reaches out for a folder this store can't answer for —
       // matching `watchProjectItem()`'s own doc comment (only `/folders/~`
       // is implemented).
-      expect(httpGet).not.toHaveBeenCalled();
+      expect(github.count()).toBe(0);
     });
   });
 });

libs/datatug/main/src/lib/nav/github-project-address.ts

@@ -181,10 +181,7 @@ export function readGithubProjectId(
 /** Thrown when an id cannot be used to read a project from GitHub. */
 export class GithubProjectIdError extends Error {
   constructor(
-    public readonly reason:
-      | GithubProjectIdProblem
-      | 'path'
-      | 'ref-not-supported',
+    public readonly reason: GithubProjectIdProblem | 'path',
     message: string,
   ) {
     super(message);

libs/datatug/main/src/lib/nav/project-url.spec.ts

@@ -976,7 +976,9 @@ describe('projectUrl refuses what it cannot write exactly (design review S2)', (
     }
     expect(counts.accepted).toBeGreaterThan(1000);
     expect(counts.refused).toBeGreaterThan(1000);
-  });
+    // About 1 s on a laptop and 3.5 to 5.6 s on a CI runner that is running the rest of the package's specs at
+    // the same time (it failed on the default 5 s timeout), so give the exhaustive walk room.
+  }, 60_000);
 
   it('is the inverse of parseProjectUrl for every other store, or refuses', () => {
     const counts = { accepted: 0, refused: 0 };

libs/datatug/main/src/lib/project-files/README.md

@@ -30,9 +30,10 @@ code yet; none of it is exported from the library barrel.
   It returns a `CheckedDataUrl` (`.href`), which nothing else makes; `tableUrls(catalog, table, trust)` does it for
   both templates of a catalog. A bare string must never reach `fetch`.
 - **The size cap is the caller's to enforce on the stream.** `parse*` measure text that is already in memory
-  (256 KB, in bytes, a leading byte order mark counted and then dropped). The fetching code (G-A2) must stop
-  reading a response once it has more than the cap, and refuse redirects; downloading the whole body and
-  measuring afterwards would not bound memory.
+  (256 KB, in bytes, a leading byte order mark counted and then dropped). The fetching code of the GitHub
+  reader (`services/repo/github/github-http.ts`, `readBodyLimited`) stops reading a response as soon as it has
+  more than the cap and cancels the transfer, and refuses redirects; downloading the whole body and measuring
+  afterwards would not bound memory.
 - **A jsDelivr pin is only as trustworthy as the trusted project's own file.** The allow-list accepts any
   40-hex commit under `datatug/chinookdb`, and GitHub serves a fork's commits through the parent
   repository's address (design 3.6). That is why the prefix is accepted for a trusted project only, whose

libs/datatug/main/src/lib/project-files/project-file-limits.ts

@@ -5,6 +5,12 @@
 /** A project file read from GitHub: at most 256 KB (3.6), counted in bytes of the text received. */
 export const MAX_PROJECT_FILE_BYTES = 256 * 1024;
 
+/** A data file a project points at (3.6): at most 5 MB, counted in bytes received. */
+export const MAX_DATA_FILE_BYTES = 5 * 1024 * 1024;
+
+/** Files one run may read from a project (3.6). */
+export const MAX_FILES_PER_RUN = 40;
+
 /** Prepared questions (5.2a). */
 export const MAX_QUESTIONS = 20;
 export const MAX_WORDINGS = 50;