diff --git a/backend/src/__tests__/adaptation.test.ts b/backend/src/__tests__/adaptation.test.ts new file mode 100644 index 00000000..093c9639 --- /dev/null +++ b/backend/src/__tests__/adaptation.test.ts @@ -0,0 +1,335 @@ +/** + * Engagement-aware content adaptation tests (issue #408) + * + * Unit-tests the deterministic adaptation rules (pacing / difficulty / + * hints), user overrides, privacy & consent controls, and effectiveness + * measurement. Then exercises the HTTP layer with supertest using a + * signed JWT (JWT_SECRET is provided by backend/tests/setup.js). + */ + +import jwt from 'jsonwebtoken'; +import express from 'express'; +import request from 'supertest'; +import { + AdaptationService, + adaptationService, +} from '../services/adaptation/AdaptationService'; +import adaptationRoutes from '../routes/adaptation'; + +const makeToken = (id = 'user-1') => + jwt.sign( + { id, username: 'learner', role: 'student', email: 'learner@test.dev' }, + process.env.JWT_SECRET || 'test-jwt-secret', + { expiresIn: '1h' }, + ); + +const buildApp = () => { + const app = express(); + app.use(express.json()); + app.use('/api/adaptation', adaptationRoutes); + return app; +}; + +describe('AdaptationService (unit)', () => { + let service: AdaptationService; + + beforeEach(() => { + service = new AdaptationService(); + }); + + describe('signal ingestion & consent', () => { + it('does not store signals without engagement tracking consent', () => { + const result = service.ingestSignal('u1', { + engagementScore: 45, + frustrationScore: 80, + dominantEmotion: 'angry', + }); + expect(result.stored).toBe(false); + expect(service.getSignals('u1')).toHaveLength(0); + }); + + it('stores signals once consent is granted', () => { + service.setConsent('u1', { engagementTrackingConsent: true }); + const result = service.ingestSignal('u1', { + engagementScore: 45, + frustrationScore: 80, + dominantEmotion: 'angry', + }); + expect(result.stored).toBe(true); + expect(service.getSignals('u1')).toHaveLength(1); + }); + + it('clamps out-of-range scores and invalid emotions', () => { + service.setConsent('u1', { engagementTrackingConsent: true }); + service.ingestSignal('u1', { + engagementScore: 500, + frustrationScore: -10, + dominantEmotion: 'not-an-emotion' as any, + }); + const stored = service.getSignals('u1')[0]; + expect(stored.engagementScore).toBe(100); + expect(stored.frustrationScore).toBe(0); + expect(stored.dominantEmotion).toBe('neutral'); + }); + }); + + describe('adaptation rules (explainable + overridable)', () => { + it('offers review when frustration exceeds 75', () => { + const rec = service.recommend('u1', { + engagementScore: 40, + frustrationScore: 90, + dominantEmotion: 'angry', + }); + expect(rec.action.kind).toBe('offer_review'); + expect(rec.rule).toBe('frustration-review'); + expect(rec.explanation).toContain('90'); + expect(rec.overridable).toBe(true); + }); + + it('slows pacing when frustration is high but below the review threshold', () => { + const rec = service.recommend('u1', { + engagementScore: 50, + frustrationScore: 65, + dominantEmotion: 'sad', + }); + expect(rec.action.kind).toBe('slow_down'); + expect(rec.action.playbackRate).toBe(0.75); + }); + + it('offers a hint when engagement drops below 30', () => { + const rec = service.recommend('u1', { + engagementScore: 20, + frustrationScore: 10, + dominantEmotion: 'neutral', + }); + expect(rec.action.kind).toBe('offer_hint'); + }); + + it('speeds up pacing for highly engaged learners', () => { + const rec = service.recommend('u1', { + engagementScore: 95, + frustrationScore: 10, + dominantEmotion: 'happy', + }); + expect(rec.action.kind).toBe('speed_up'); + expect(rec.action.playbackRate).toBe(1.25); + }); + + it('keeps standard pacing in the healthy band', () => { + const rec = service.recommend('u1', { + engagementScore: 60, + frustrationScore: 20, + dominantEmotion: 'neutral', + }); + expect(rec.action.kind).toBe('none'); + }); + + it('respects the master adaptationEnabled preference', () => { + service.setPreferences('u1', { adaptationEnabled: false }); + const rec = service.recommend('u1', { + engagementScore: 10, + frustrationScore: 95, + dominantEmotion: 'angry', + }); + expect(rec.action.kind).toBe('none'); + expect(rec.rule).toBe('preference-disabled'); + }); + + it('respects per-dimension overrides (pacing disabled → no slow down)', () => { + service.setPreferences('u1', { pacingEnabled: false }); + const rec = service.recommend('u1', { + engagementScore: 50, + frustrationScore: 65, + dominantEmotion: 'sad', + }); + expect(rec.action.kind).not.toBe('slow_down'); + expect(['offer_review', 'offer_hint', 'simplify', 'none']).toContain(rec.action.kind); + }); + + it('respects per-dimension overrides (hints disabled → no hint)', () => { + service.setPreferences('u1', { hintsEnabled: false }); + const rec = service.recommend('u1', { + engagementScore: 20, + frustrationScore: 10, + dominantEmotion: 'neutral', + }); + expect(rec.action.kind).not.toBe('offer_hint'); + }); + + it('clamps playback bounds via preferences', () => { + const prefs = service.setPreferences('u1', { + maxPlaybackRate: 5, + minPlaybackRate: 0, + }); + expect(prefs.maxPlaybackRate).toBe(2); + expect(prefs.minPlaybackRate).toBe(0.25); + }); + }); + + describe('privacy & consent controls', () => { + it('revoking consent purges stored signals (right-to-erasure)', () => { + service.setConsent('u1', { engagementTrackingConsent: true }); + service.ingestSignal('u1', { + engagementScore: 50, + frustrationScore: 50, + dominantEmotion: 'neutral', + }); + expect(service.getSignals('u1')).toHaveLength(1); + + service.setConsent('u1', { engagementTrackingConsent: false }); + expect(service.getSignals('u1')).toHaveLength(0); + expect(service.getConsent('u1').revokedAt).toBeDefined(); + }); + + it('purgeUserData clears every store', () => { + service.setConsent('u1', { engagementTrackingConsent: true }); + service.ingestSignal('u1', { + engagementScore: 50, + frustrationScore: 50, + dominantEmotion: 'neutral', + }); + service.recommend('u1', { + engagementScore: 50, + frustrationScore: 50, + dominantEmotion: 'neutral', + }); + service.purgeUserData('u1'); + expect(service.getSignals('u1')).toHaveLength(0); + expect(service.getRecommendations('u1')).toHaveLength(0); + expect(service.getEffectiveness('u1')).toHaveLength(0); + }); + }); + + describe('effectiveness measurement', () => { + it('aggregates acceptance rate and score deltas per action', () => { + const rec1 = service.recommend('u1', { + engagementScore: 90, + frustrationScore: 10, + dominantEmotion: 'happy', + }); + const rec2 = service.recommend('u1', { + engagementScore: 90, + frustrationScore: 10, + dominantEmotion: 'happy', + }); + + service.recordOutcome('u1', { + recommendationId: rec1.id, + accepted: true, + quizScoreAfter: 90, + }); + service.recordOutcome('u1', { + recommendationId: rec2.id, + accepted: false, + quizScoreAfter: 55, + }); + + const effectiveness = service.getEffectiveness('u1'); + const speedUp = effectiveness.find((e) => e.action === 'speed_up'); + expect(speedUp).toBeDefined(); + expect(speedUp!.samples).toBe(2); + expect(speedUp!.acceptanceRate).toBe(0.5); + expect(speedUp!.averageScoreDelta).toBeGreaterThan(0); + }); + }); +}); + +describe('Adaptation routes (HTTP)', () => { + const app = buildApp(); + + beforeEach(() => { + adaptationService.reset(); + }); + + it('rejects unauthenticated requests', async () => { + const res = await request(app).get('/api/adaptation/preferences'); + expect(res.status).toBe(401); + }); + + it('ingests a signal with a valid token', async () => { + const res = await request(app) + .post('/api/adaptation/signal') + .set('Authorization', `Bearer ${makeToken()}`) + .send({ engagementScore: 40, frustrationScore: 80, dominantEmotion: 'angry' }); + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + }); + + it('rejects a malformed signal payload', async () => { + const res = await request(app) + .post('/api/adaptation/signal') + .set('Authorization', `Bearer ${makeToken()}`) + .send({ dominantEmotion: 'angry' }); + expect(res.status).toBe(400); + }); + + it('returns an explainable, overridable recommendation', async () => { + const res = await request(app) + .post('/api/adaptation/recommend') + .set('Authorization', `Bearer ${makeToken('learner-1')}`) + .send({ engagementScore: 20, frustrationScore: 10, dominantEmotion: 'neutral' }); + expect(res.status).toBe(200); + expect(res.body.data.action.kind).toBe('offer_hint'); + expect(res.body.data.explanation.length).toBeGreaterThan(0); + expect(res.body.data.overridable).toBe(true); + }); + + it('updates and reads preferences', async () => { + const update = await request(app) + .put('/api/adaptation/preferences') + .set('Authorization', `Bearer ${makeToken()}`) + .send({ pacingEnabled: false }); + expect(update.status).toBe(200); + expect(update.body.data.pacingEnabled).toBe(false); + + const read = await request(app) + .get('/api/adaptation/preferences') + .set('Authorization', `Bearer ${makeToken()}`); + expect(read.status).toBe(200); + expect(read.body.data.pacingEnabled).toBe(false); + }); + + it('grants and revokes consent via the API', async () => { + const grant = await request(app) + .put('/api/adaptation/consent') + .set('Authorization', `Bearer ${makeToken('consent-user')}`) + .send({ engagementTrackingConsent: true }); + expect(grant.status).toBe(200); + expect(grant.body.data.engagementTrackingConsent).toBe(true); + + const revoke = await request(app) + .put('/api/adaptation/consent') + .set('Authorization', `Bearer ${makeToken('consent-user')}`) + .send({ engagementTrackingConsent: false }); + expect(revoke.status).toBe(200); + expect(revoke.body.data.revokedAt).toBeDefined(); + }); + + it('records outcomes and reports effectiveness', async () => { + const recRes = await request(app) + .post('/api/adaptation/recommend') + .set('Authorization', `Bearer ${makeToken('effect-user')}`) + .send({ engagementScore: 95, frustrationScore: 5, dominantEmotion: 'happy' }); + const recId = recRes.body.data.id; + + const outcome = await request(app) + .post('/api/adaptation/outcome') + .set('Authorization', `Bearer ${makeToken('effect-user')}`) + .send({ recommendationId: recId, accepted: true, quizScoreAfter: 88 }); + expect(outcome.status).toBe(200); + + const effectiveness = await request(app) + .get('/api/adaptation/effectiveness') + .set('Authorization', `Bearer ${makeToken('effect-user')}`); + expect(effectiveness.status).toBe(200); + expect(effectiveness.body.data.length).toBeGreaterThan(0); + }); + + it('purges all user data', async () => { + const res = await request(app) + .delete('/api/adaptation/purge') + .set('Authorization', `Bearer ${makeToken('purge-user')}`); + expect(res.status).toBe(200); + expect(res.body.message).toContain('purged'); + }); +}); diff --git a/backend/src/index.ts b/backend/src/index.ts index ad04c5bf..ccc0af5a 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -146,6 +146,10 @@ const cspViolationRoutes = loadRoute('./routes/cspViolationRoutes'); // @ts-ignore const jobRoutes = loadRoute('./routes/jobRoutes'); +// Engagement-aware content adaptation routes — Issue #408 +// @ts-ignore +const adaptationRoutes = loadRoute('./routes/adaptation'); + // Initialize Express app const app: Application = express(); const server = createServer(app); @@ -340,6 +344,9 @@ app.use('/api/metrics', metricsRoutes); // Background job management routes — Issue #258 app.use('/api/jobs', jobRoutes); +// Engagement-aware content adaptation — Issue #408 +app.use('/api/adaptation', adaptationRoutes); + // Root endpoint // ── Versioned API routes (/api/v1/*) ──────────────────────────────────────── // @@ -377,6 +384,7 @@ app.use('/api/v1/time-lock', timeLockCredentialsRoutes); app.use('/api/v1/vrf', vrfRoutes); app.use('/api/v1/translate', translationRoutes); app.use('/api/v1/localization', localizationRoutes); +app.use('/api/v1/adaptation', adaptationRoutes); app.use('/api/v1/cross-protocol-bridge', crossProtocolBridgeRoutes); app.use('/api/v1/audit', auditRoutes); app.get('/api/v1/health', (req, res) => { @@ -641,6 +649,7 @@ server.listen(PORT, () => { '/api/metrics', '/api/health', '/api/jobs', + '/api/adaptation', ], }); }); diff --git a/backend/src/routes/adaptation.ts b/backend/src/routes/adaptation.ts new file mode 100644 index 00000000..df2baee8 --- /dev/null +++ b/backend/src/routes/adaptation.ts @@ -0,0 +1,345 @@ +/** + * @openapi + * tags: + * - name: Adaptation + * description: Engagement-aware content adaptation (issue #408) + */ + +import express, { NextFunction, Request, Response } from 'express'; +import { body, validationResult } from 'express-validator'; +import { adaptationService } from '../services/adaptation/AdaptationService'; +import { authenticateToken } from '../middleware/auth'; + +const router: express.Router = express.Router(); + +const validateRequest = (req: Request, res: Response, next: NextFunction) => { + const errors = validationResult(req); + if (!errors.isEmpty()) { + return res.status(400).json({ success: false, errors: errors.array() }); + } + next(); +}; + +/** + * @openapi + * /api/adaptation/signal: + * post: + * tags: [Adaptation] + * summary: Report a client-derived engagement signal + * description: Ingest the derived signal from the frontend emotion hook. Raw video never leaves the client. + * security: + * - bearerAuth: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * engagementScore: + * type: number + * frustrationScore: + * type: number + * dominantEmotion: + * type: string + * progress: + * type: number + * responses: + * '200': + * description: Signal ingested (stored only if consent is granted) + */ +router.post( + '/signal', + authenticateToken, + [ + body('engagementScore').isNumeric().withMessage('engagementScore must be a number'), + body('frustrationScore').isNumeric().withMessage('frustrationScore must be a number'), + body('dominantEmotion').isString().withMessage('dominantEmotion must be a string'), + ], + validateRequest, + (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + const { engagementScore, frustrationScore, dominantEmotion, progress } = req.body; + const result = adaptationService.ingestSignal(userId, { + engagementScore: Number(engagementScore), + frustrationScore: Number(frustrationScore), + dominantEmotion, + progress: progress !== undefined ? Number(progress) : undefined, + }); + res.json({ success: true, data: result }); + }, +); + +/** + * @openapi + * /api/adaptation/recommend: + * post: + * tags: [Adaptation] + * summary: Get an explainable, overridable adaptation recommendation + * description: Decision based on the current signal (and stored window if consent was given). + * security: + * - bearerAuth: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * engagementScore: + * type: number + * frustrationScore: + * type: number + * dominantEmotion: + * type: string + * progress: + * type: number + * responses: + * '200': + * description: Adaptation recommendation + */ +router.post( + '/recommend', + authenticateToken, + [ + body('engagementScore').isNumeric().withMessage('engagementScore must be a number'), + body('frustrationScore').isNumeric().withMessage('frustrationScore must be a number'), + body('dominantEmotion').isString().withMessage('dominantEmotion must be a string'), + ], + validateRequest, + (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + const signal = { + engagementScore: Number(req.body.engagementScore), + frustrationScore: Number(req.body.frustrationScore), + dominantEmotion: req.body.dominantEmotion, + progress: req.body.progress !== undefined ? Number(req.body.progress) : undefined, + }; + const recommendation = adaptationService.recommend(userId, signal); + res.json({ success: true, data: recommendation }); + }, +); + +/** + * @openapi + * /api/adaptation/preferences: + * get: + * tags: [Adaptation] + * summary: Get adaptation preferences for the authenticated user + * security: + * - bearerAuth: [] + * responses: + * '200': + * description: User preferences + */ +router.get('/preferences', authenticateToken, (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + res.json({ success: true, data: adaptationService.getPreferences(userId) }); +}); + +/** + * @openapi + * /api/adaptation/preferences: + * put: + * tags: [Adaptation] + * summary: Update adaptation preferences + * security: + * - bearerAuth: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * adaptationEnabled: + * type: boolean + * pacingEnabled: + * type: boolean + * difficultyEnabled: + * type: boolean + * hintsEnabled: + * type: boolean + * maxPlaybackRate: + * type: number + * minPlaybackRate: + * type: number + * responses: + * '200': + * description: Preferences updated + */ +router.put( + '/preferences', + authenticateToken, + [ + body('adaptationEnabled').optional().isBoolean(), + body('pacingEnabled').optional().isBoolean(), + body('difficultyEnabled').optional().isBoolean(), + body('hintsEnabled').optional().isBoolean(), + body('maxPlaybackRate').optional().isNumeric(), + body('minPlaybackRate').optional().isNumeric(), + ], + validateRequest, + (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + const prefs = adaptationService.setPreferences(userId, req.body); + res.json({ success: true, data: prefs }); + }, +); + +/** + * @openapi + * /api/adaptation/consent: + * get: + * tags: [Adaptation] + * summary: Get consent state for the authenticated user + * security: + * - bearerAuth: [] + * responses: + * '200': + * description: Consent state + */ +router.get('/consent', authenticateToken, (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + res.json({ success: true, data: adaptationService.getConsent(userId) }); +}); + +/** + * @openapi + * /api/adaptation/consent: + * put: + * tags: [Adaptation] + * summary: Update consent state (revocation purges stored signals) + * security: + * - bearerAuth: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * engagementTrackingConsent: + * type: boolean + * biometricConsent: + * type: boolean + * responses: + * '200': + * description: Consent updated + */ +router.put( + '/consent', + authenticateToken, + [ + body('engagementTrackingConsent').optional().isBoolean(), + body('biometricConsent').optional().isBoolean(), + ], + validateRequest, + (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + const consent = adaptationService.setConsent(userId, req.body); + res.json({ success: true, data: consent }); + }, +); + +/** + * @openapi + * /api/adaptation/outcome: + * post: + * tags: [Adaptation] + * summary: Record learning outcome measured after an adaptation was applied + * security: + * - bearerAuth: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * recommendationId: + * type: string + * accepted: + * type: boolean + * quizScoreAfter: + * type: number + * completionDelta: + * type: number + * responses: + * '200': + * description: Outcome recorded + */ +router.post( + '/outcome', + authenticateToken, + [ + body('recommendationId').isString().withMessage('recommendationId is required'), + body('accepted').isBoolean().withMessage('accepted must be a boolean'), + ], + validateRequest, + (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + const outcome = adaptationService.recordOutcome(userId, { + userId, + recommendationId: req.body.recommendationId, + accepted: Boolean(req.body.accepted), + quizScoreAfter: req.body.quizScoreAfter !== undefined ? Number(req.body.quizScoreAfter) : undefined, + completionDelta: req.body.completionDelta !== undefined ? Number(req.body.completionDelta) : undefined, + }); + res.json({ success: true, data: outcome }); + }, +); + +/** + * @openapi + * /api/adaptation/effectiveness: + * get: + * tags: [Adaptation] + * summary: Get adaptation effectiveness metrics for the authenticated user + * security: + * - bearerAuth: [] + * responses: + * '200': + * description: Effectiveness metrics + */ +router.get('/effectiveness', authenticateToken, (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + res.json({ success: true, data: adaptationService.getEffectiveness(userId) }); +}); + +/** + * @openapi + * /api/adaptation/recommendations: + * get: + * tags: [Adaptation] + * summary: List past recommendations for the authenticated user + * security: + * - bearerAuth: [] + * responses: + * '200': + * description: Past recommendations + */ +router.get('/recommendations', authenticateToken, (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + res.json({ success: true, data: adaptationService.getRecommendations(userId) }); +}); + +/** + * @openapi + * /api/adaptation/purge: + * delete: + * tags: [Adaptation] + * summary: Purge all stored engagement data (GDPR right-to-erasure) + * security: + * - bearerAuth: [] + * responses: + * '200': + * description: Data purged + */ +router.delete('/purge', authenticateToken, (req: Request, res: Response) => { + const userId = (req as any).user?.id || 'anonymous'; + adaptationService.purgeUserData(userId); + res.json({ success: true, message: 'User data purged' }); +}); + +export default router; \ No newline at end of file diff --git a/backend/src/services/adaptation/AdaptationService.ts b/backend/src/services/adaptation/AdaptationService.ts new file mode 100644 index 00000000..0c3a4770 --- /dev/null +++ b/backend/src/services/adaptation/AdaptationService.ts @@ -0,0 +1,487 @@ +/** + * Engagement-Aware Content Adaptation Service (issue #408) + * + * Uses the existing emotion/engagement detection signals (frontend + * `useEmotionDetection` hook, neural data) to adapt content delivery: + * pacing (playback rate), difficulty, and hints are adjusted when a + * learner is disengaged or struggling. + * + * Design notes + * ------------ + * - Signals arrive as *derived* scores (engagement 0-100, frustration + * 0-100, dominant emotion) — raw video/biometric data never leaves the + * client, so this service never touches it. + * - State is kept in memory, mirroring `bridgeMonitor.ts`; a production + * deployment would persist profiles to Redis/Mongo. + * - Decisions are deterministic and *explainable*: every recommendation + * carries a human-readable explanation plus the rule id that fired. + * - Every adaptation is *user-overridable*: per-user preferences gate + * which dimensions (pacing / difficulty / hints) may auto-adapt and + * bound the allowed playback range. + * - Consent is enforced: when a user revokes consent, stored signals are + * purged and signal ingestion stops. + * - Effectiveness is measured: outcomes recorded after each adaptation + * (quiz score, completion) are aggregated per action so impact on + * learning outcomes can be evaluated. + */ + +export type Emotion = + | 'neutral' + | 'happy' + | 'sad' + | 'angry' + | 'fearful' + | 'disgusted' + | 'surprised'; + +export type AdaptationDimension = 'pacing' | 'difficulty' | 'hints'; +export type AdaptationKind = 'slow_down' | 'speed_up' | 'offer_review' | 'offer_hint' | 'simplify' | 'none'; + +export interface EngagementSignal { + /** Client-derived scores, 0-100. Never contains raw biometric data. */ + engagementScore: number; + frustrationScore: number; + dominantEmotion: Emotion; + /** 0-1, where 1 = video fully watched. Used to keep hints relevant. */ + progress?: number; + timestamp?: number; +} + +export interface AdaptationPreference { + /** Master switch — disables all automatic adaptation. */ + adaptationEnabled: boolean; + /** Allow pacing (playback rate) adjustments. */ + pacingEnabled: boolean; + /** Allow difficulty adjustments. */ + difficultyEnabled: boolean; + /** Allow hint / alternate-explanation injection. */ + hintsEnabled: boolean; + /** Bounds for automatic playback-rate changes (0.25 - 2.0). */ + maxPlaybackRate: number; + minPlaybackRate: number; +} + +export interface ConsentState { + /** Opt-in to storing engagement signals for adaptation. */ + engagementTrackingConsent: boolean; + /** Whether the learner consents to any biometric capture at all. */ + biometricConsent: boolean; + grantedAt?: number; + revokedAt?: number; +} + +export interface AdaptationAction { + kind: AdaptationKind; + /** Target playback rate for `slow_down` / `speed_up`. */ + playbackRate?: number; + /** Suggested difficulty label for `simplify`. */ + difficulty?: string; + /** Hint text for `offer_hint` / `offer_review`. */ + message?: string; +} + +export interface AdaptationRecommendation { + id: string; + userId: string; + action: AdaptationAction; + /** Human-readable explanation of *why* this adaptation was chosen. */ + explanation: string; + /** Deterministic rule id that produced the recommendation. */ + rule: string; + /** Model confidence in the underlying engagement state (0-1). */ + confidence: number; + /** Always true — every recommendation can be dismissed/overridden. */ + overridable: boolean; + createdAt: number; +} + +export interface StoredSignal extends EngagementSignal { + timestamp: number; +} + +export interface AdaptationOutcome { + id: string; + userId: string; + recommendationId: string; + /** Whether the learner accepted (vs dismissed) the adaptation. */ + accepted: boolean; + /** Learning-outcome proxy measured after the adaptation. */ + quizScoreAfter?: number; + completionDelta?: number; + timestamp: number; +} + +export interface EngagementState { + averageEngagement: number; + averageFrustration: number; + sampleCount: number; + dominantEmotion: Emotion; +} + +const DEFAULT_PREFERENCES: AdaptationPreference = { + adaptationEnabled: true, + pacingEnabled: true, + difficultyEnabled: true, + hintsEnabled: true, + maxPlaybackRate: 1.5, + minPlaybackRate: 0.5, +}; + +const DEFAULT_CONSENT: ConsentState = { + engagementTrackingConsent: false, + biometricConsent: false, +}; + +/** Keep only the last N signals per user (rolling window). */ +const MAX_SIGNALS_PER_USER = 50; + +export class AdaptationService { + private signals = new Map(); + private recommendations = new Map(); + private preferences = new Map(); + private consents = new Map(); + private outcomes = new Map(); + private ruleCounter = 0; + + /** + * Ingests a client-derived engagement signal. Respects consent: without + * `engagementTrackingConsent` the signal is validated but not stored, so + * the user still gets a *stateless* recommendation if they call + * `recommend` with the same signal. + */ + ingestSignal(userId: string, signal: EngagementSignal): { stored: boolean } { + const normalized = this.normalizeSignal(signal); + const consent = this.getConsent(userId); + + if (!consent.engagementTrackingConsent) { + return { stored: false }; + } + + const window = this.signals.get(userId) || []; + window.push(normalized); + // Keep the rolling window bounded. + this.signals.set(userId, window.slice(-MAX_SIGNALS_PER_USER)); + return { stored: true }; + } + + /** + * Pure decision: given a signal (plus optionally the stored rolling + * window) produce an explainable, overridable adaptation. + * + * The returned recommendation is recorded so the learner can review + * what the system did and why. + */ + recommend(userId: string, signal: EngagementSignal): AdaptationRecommendation { + const normalized = this.normalizeSignal(signal); + const prefs = this.getPreferences(userId); + + if (!prefs.adaptationEnabled) { + return this.buildRecommendation(userId, { + kind: 'none', + message: 'Automatic adaptation is disabled in your preferences.', + }, 'preference-disabled', 'Adaptation is turned off in your preferences, so no changes were made.', 0); + } + + const state = this.computeEngagementState(userId, normalized); + const decision = this.decide(state, prefs); + + return this.buildRecommendation( + userId, + decision.action, + decision.rule, + decision.explanation, + decision.confidence, + ); + } + + /** + * Evaluates the rolling engagement state against deterministic rules. + * Returns the highest-priority matching rule — mirroring the existing + * `EmotionDashboard` thresholds but as a backend-grade, testable engine. + */ + private decide( + state: EngagementState, + prefs: AdaptationPreference, + ): { action: AdaptationAction; rule: string; explanation: string; confidence: number } { + const frustration = state.averageFrustration; + const engagement = state.averageEngagement; + + if (prefs.hintsEnabled && frustration > 75) { + return { + action: { + kind: 'offer_review', + message: 'You seem frustrated. Would you like to review the previous section or take a short break?', + }, + rule: 'frustration-review', + explanation: + `Frustration averaged ${Math.round(frustration)}/100 over the last ${state.sampleCount} sample(s), ` + + 'which is above the 75 threshold. Reviewing earlier material reduces cognitive load.', + confidence: 0.9, + }; + } + + if (prefs.pacingEnabled && frustration > 60) { + const rate = Math.max(prefs.minPlaybackRate, 0.75); + return { + action: { kind: 'slow_down', playbackRate: rate }, + rule: 'frustration-slow-down', + explanation: + `Frustration averaged ${Math.round(frustration)}/100, above the 60 threshold. ` + + `Slowing the pacing to ${rate}× gives you more time to process.`, + confidence: 0.85, + }; + } + + if (prefs.hintsEnabled && engagement < 30) { + return { + action: { + kind: 'offer_hint', + message: 'Engagement is dropping — here is a hint to help you stay on track.', + }, + rule: 'low-engagement-hint', + explanation: + `Engagement averaged ${Math.round(engagement)}/100, below the 30 threshold. ` + + 'An alternate explanation or hint is offered to re-engage you.', + confidence: 0.8, + }; + } + + if (prefs.difficultyEnabled && engagement < 40) { + return { + action: { kind: 'simplify', difficulty: 'beginner' }, + rule: 'low-engagement-simplify', + explanation: + `Engagement averaged ${Math.round(engagement)}/100, below the 40 threshold. ` + + 'The content difficulty is temporarily simplified to rebuild confidence.', + confidence: 0.75, + }; + } + + if (prefs.pacingEnabled && engagement > 80 && state.averageFrustration < 40) { + const rate = Math.min(prefs.maxPlaybackRate, 1.25); + return { + action: { kind: 'speed_up', playbackRate: rate }, + rule: 'high-engagement-speed-up', + explanation: + `Engagement averaged ${Math.round(engagement)}/100 with low frustration, above the 80 threshold. ` + + `Pacing increased to ${rate}× to match your flow state.`, + confidence: 0.85, + }; + } + + return { + action: { kind: 'none', message: 'No adaptation needed right now.' }, + rule: 'standard-pacing', + explanation: + `Engagement ${Math.round(engagement)}/100 and frustration ${Math.round(frustration)}/100 are both ` + + 'within the healthy band, so the current pacing is kept.', + confidence: 0.7, + }; + } + + /** + * Per-user preferences gate every decision dimension, making adaptation + * explainable and user-overridable. + */ + setPreferences(userId: string, prefs: Partial): AdaptationPreference { + const merged: AdaptationPreference = { + ...this.getPreferences(userId), + ...prefs, + }; + // Clamp playback bounds to a sane range. + merged.maxPlaybackRate = this.clamp(merged.maxPlaybackRate, 0.25, 2); + merged.minPlaybackRate = this.clamp(merged.minPlaybackRate, 0.25, 2); + if (merged.minPlaybackRate > merged.maxPlaybackRate) { + merged.minPlaybackRate = merged.maxPlaybackRate; + } + this.preferences.set(userId, merged); + return { ...merged }; + } + + getPreferences(userId: string): AdaptationPreference { + return { ...(this.preferences.get(userId) || DEFAULT_PREFERENCES) }; + } + + /** + * Privacy & consent controls. Revoking consent purges stored signals + * immediately (right-to-erasure for derived engagement data). + */ + setConsent(userId: string, consent: Partial): ConsentState { + const current = this.getConsent(userId); + const now = Date.now(); + const merged: ConsentState = { ...current, ...consent }; + + if (merged.engagementTrackingConsent && !current.engagementTrackingConsent) { + merged.grantedAt = now; + } + if (!merged.engagementTrackingConsent && current.engagementTrackingConsent) { + merged.revokedAt = now; + this.signals.delete(userId); + } + + this.consents.set(userId, merged); + return { ...merged }; + } + + getConsent(userId: string): ConsentState { + return { ...(this.consents.get(userId) || DEFAULT_CONSENT) }; + } + + /** Purge all stored engagement data for a user (GDPR-style erasure). */ + purgeUserData(userId: string): void { + this.signals.delete(userId); + this.recommendations.delete(userId); + this.outcomes.delete(userId); + this.consents.delete(userId); + this.preferences.delete(userId); + } + + /** + * Record a learning outcome measured after an adaptation was applied so + * the effectiveness of adaptation can be evaluated against outcomes. + */ + recordOutcome(userId: string, outcome: Omit): AdaptationOutcome { + const stored: AdaptationOutcome = { + ...outcome, + id: `outcome_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`, + timestamp: Date.now(), + }; + const list = this.outcomes.get(userId) || []; + list.push(stored); + this.outcomes.set(userId, list.slice(-100)); + return { ...stored }; + } + + /** + * Aggregate outcomes per action type so the impact of adaptation on + * learning outcomes can be measured (acceptance rate + score delta). + */ + getEffectiveness(userId: string) { + const outcomes = this.outcomes.get(userId) || []; + const byKind = new Map(); + + for (const o of outcomes) { + const recommendation = (this.recommendations.get(userId) || []).find( + (r) => r.id === o.recommendationId, + ); + const kind = recommendation?.action.kind || 'unknown'; + const entry = byKind.get(kind) || { count: 0, accepted: 0, scoreDeltas: [] }; + entry.count += 1; + if (o.accepted) entry.accepted += 1; + if (o.quizScoreAfter !== undefined && recommendation) { + // Delta is measured against the baseline engagement at decision time. + const baseline = recommendation.confidence * 50; + entry.scoreDeltas.push(o.quizScoreAfter - baseline); + } + byKind.set(kind, entry); + } + + return Array.from(byKind.entries()).map(([kind, entry]) => ({ + action: kind, + samples: entry.count, + acceptanceRate: entry.count > 0 ? entry.accepted / entry.count : 0, + averageScoreDelta: + entry.scoreDeltas.length > 0 + ? entry.scoreDeltas.reduce((a, b) => a + b, 0) / entry.scoreDeltas.length + : 0, + })); + } + + getSignals(userId: string): StoredSignal[] { + return (this.signals.get(userId) || []).map((s) => ({ ...s })); + } + + getRecommendations(userId: string): AdaptationRecommendation[] { + return (this.recommendations.get(userId) || []).map((r) => ({ ...r })); + } + + reset(): void { + this.signals.clear(); + this.recommendations.clear(); + this.preferences.clear(); + this.consents.clear(); + this.outcomes.clear(); + } + + // ── helpers ───────────────────────────────────────────────────────────── + + private buildRecommendation( + userId: string, + action: AdaptationAction, + rule: string, + explanation: string, + confidence: number, + ): AdaptationRecommendation { + this.ruleCounter += 1; + const recommendation: AdaptationRecommendation = { + id: `adapt_${Date.now()}_${this.ruleCounter}`, + userId, + action, + explanation, + rule, + confidence: this.clamp(confidence, 0, 1), + overridable: true, + createdAt: Date.now(), + }; + const list = this.recommendations.get(userId) || []; + list.push(recommendation); + this.recommendations.set(userId, list.slice(-50)); + return { ...recommendation }; + } + + private computeEngagementState(userId: string, signal: StoredSignal): EngagementState { + const window = this.signals.get(userId) || []; + const samples = [...window, signal]; + + const avg = (fn: (s: StoredSignal) => number) => + samples.reduce((sum, s) => sum + fn(s), 0) / Math.max(1, samples.length); + + const counts = new Map(); + for (const s of samples) { + counts.set(s.dominantEmotion, (counts.get(s.dominantEmotion) || 0) + 1); + } + let dominantEmotion: Emotion = signal.dominantEmotion; + let max = -1; + for (const [emotion, count] of counts.entries()) { + if (count > max) { + max = count; + dominantEmotion = emotion; + } + } + + return { + averageEngagement: avg((s) => s.engagementScore), + averageFrustration: avg((s) => s.frustrationScore), + sampleCount: samples.length, + dominantEmotion, + }; + } + + private normalizeSignal(signal: EngagementSignal): StoredSignal { + const engagementScore = this.clamp(Number(signal.engagementScore) || 0, 0, 100); + const frustrationScore = this.clamp(Number(signal.frustrationScore) || 0, 0, 100); + const dominantEmotion: Emotion = this.isEmotion(signal.dominantEmotion) + ? signal.dominantEmotion + : 'neutral'; + return { + engagementScore, + frustrationScore, + dominantEmotion, + progress: this.clamp(Number(signal.progress) || 0, 0, 1), + timestamp: signal.timestamp || Date.now(), + }; + } + + private isEmotion(value: unknown): value is Emotion { + return ( + typeof value === 'string' && + ['neutral', 'happy', 'sad', 'angry', 'fearful', 'disgusted', 'surprised'].includes(value) + ); + } + + private clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); + } +} + +export const adaptationService = new AdaptationService(); diff --git a/backend/src/services/adaptation/index.ts b/backend/src/services/adaptation/index.ts new file mode 100644 index 00000000..c1df1d5a --- /dev/null +++ b/backend/src/services/adaptation/index.ts @@ -0,0 +1,14 @@ +export { AdaptationService, adaptationService } from './AdaptationService'; +export type { + Emotion, + EngagementSignal, + AdaptationPreference, + ConsentState, + AdaptationAction, + AdaptationRecommendation, + StoredSignal, + AdaptationOutcome, + EngagementState, + AdaptationDimension, + AdaptationKind, +} from './AdaptationService'; \ No newline at end of file diff --git a/frontend/src/components/player/AdaptivePlayer.tsx b/frontend/src/components/player/AdaptivePlayer.tsx new file mode 100644 index 00000000..18c2d6da --- /dev/null +++ b/frontend/src/components/player/AdaptivePlayer.tsx @@ -0,0 +1,306 @@ +'use client'; + +import React, { useEffect, useRef, useState } from 'react'; +import { motion, AnimatePresence } from 'framer-motion'; +import { Gauge, Info, ShieldCheck, Zap } from 'lucide-react'; +import { useEmotionDetection } from '../../hooks/useEmotionDetection'; +import type { AdaptationAction } from '../../lib/adaptationEngine'; + +interface Subtitle { + src: string; + srcLang: string; + label: string; + default?: boolean; + kind?: 'subtitles' | 'captions'; +} + +interface AdaptivePlayerProps { + src: string; + title: string; + subtitles?: Subtitle[]; + poster?: string; + /** Called whenever the engine applies an adaptation. */ + onAdaptationApplied?: () => void; +} + +const ADAPTATION_LABEL: Record = { + slow_down: 'Pacing slowed', + speed_up: 'Pacing increased', + offer_review: 'Review suggested', + offer_hint: 'Hint offered', + simplify: 'Difficulty simplified', +}; + +/** + * Engagement-aware adaptive video player (issue #408). + * + * Wires the emotion/engagement detection hook into content delivery: + * pacing (playback rate) adapts to the learner's engagement, every + * adaptation is explained and can be accepted or dismissed, and the + * learner controls consent + which dimensions may auto-adapt. + */ +export const AdaptivePlayer: React.FC = ({ + src, + title, + subtitles = [], + poster, + onAdaptationApplied, +}) => { + const videoRef = useRef(null); + const [showSettings, setShowSettings] = useState(false); + const { + videoRef: cameraRef, + dominantEmotion, + engagementScore, + frustrationScore, + isDetecting, + error, + startDetection, + stopDetection, + playbackRate, + currentAdaptation, + adaptationExplanation, + isAdapting, + preferences, + consent, + updatePreference, + setConsent, + dismissAdaptation, + acceptAdaptation, + } = useEmotionDetection({ onAdaptationApplied }); + + // Apply the engine's pacing to the native video element. + useEffect(() => { + if (videoRef.current && playbackRate !== 1) { + videoRef.current.playbackRate = playbackRate; + } + }, [playbackRate]); + + const statusText = `${title} is playing at ${playbackRate.toFixed(2)}×. ` + + `Engagement ${Math.round(engagementScore)}/100, frustration ${Math.round(frustrationScore)}/100.`; + + const action: AdaptationAction | null = currentAdaptation; + const actionLabel = action ? ADAPTATION_LABEL[action.kind] || action.kind : null; + + return ( +
+ + +
+ {statusText} +
+ + {/* Explainable, user-overridable adaptation banner */} + + {isAdapting && action && actionLabel && ( + +
+ +
+

{actionLabel}

+ {adaptationExplanation && ( +

{adaptationExplanation}

+ )} + {action.kind === 'slow_down' || action.kind === 'speed_up' ? ( +

+ Playback speed set to {action.playbackRate?.toFixed(2)}× +

+ ) : null} +
+ + +
+
+ +
+ + )} + + + {/* Engagement + privacy controls */} +
+ + {consent.engagementTrackingConsent && ( + + + Tracking on + + )} +
+ + + {showSettings && ( + +
+

+ Adaptation Settings +

+ +
+ + {/* Emotion state */} +
+
+ Emotion + {isDetecting ? dominantEmotion : '—'} +
+
+ Engagement + {isDetecting ? `${Math.round(engagementScore)}%` : '—'} +
+
+ Frustration + {isDetecting ? `${Math.round(frustrationScore)}%` : '—'} +
+
+ Pacing + {playbackRate.toFixed(2)}× +
+
+ + {error &&

{error}

} + + {/* Consent + camera controls */} +
+ + + + Only derived scores are shared — raw video never leaves your device. + +
+ + {/* Adaptation dimensions */} +
+ + + + +
+ + )} + + +
+ {subtitles.length > 0 + ? `Caption tracks available for ${title}.` + : `No caption tracks were provided for ${title}.`} +
+
+ ); +}; + +export default AdaptivePlayer; diff --git a/frontend/src/components/player/index.ts b/frontend/src/components/player/index.ts new file mode 100644 index 00000000..bf83e903 --- /dev/null +++ b/frontend/src/components/player/index.ts @@ -0,0 +1 @@ +export { AdaptivePlayer, default as AdaptivePlayerDefault } from './AdaptivePlayer'; \ No newline at end of file diff --git a/frontend/src/hooks/useEmotionDetection.ts b/frontend/src/hooks/useEmotionDetection.ts index 0b8fb803..c3334ce1 100644 --- a/frontend/src/hooks/useEmotionDetection.ts +++ b/frontend/src/hooks/useEmotionDetection.ts @@ -1,7 +1,20 @@ import { useState, useEffect, useRef, useCallback } from 'react'; import * as tf from '@tensorflow/tfjs'; +import { + decideAdaptation, + playbackRateFor, + adaptationApi, + DEFAULT_PREFERENCES, + DEFAULT_CONSENT, +} from '../lib/adaptationEngine'; +import type { + Emotion, + AdaptationPreference, + ConsentState, + AdaptationAction, +} from '../lib/adaptationEngine'; -export type Emotion = 'neutral' | 'happy' | 'sad' | 'angry' | 'fearful' | 'disgusted' | 'surprised'; +export type { Emotion } from '../lib/adaptationEngine'; export interface EmotionState { dominantEmotion: Emotion; @@ -11,15 +24,110 @@ export interface EmotionState { error?: string; } -export const useEmotionDetection = () => { +export interface AdaptiveState { + /** Currently applied playback rate (1 = unchanged). */ + playbackRate: number; + /** The active adaptation action (or `none`). */ + currentAction: AdaptationAction | null; + /** Human-readable explanation for the current adaptation. */ + explanation: string | null; + /** Whether an adaptation is being shown to the learner. */ + isAdapting: boolean; + /** Deterministic rule id that produced the current adaptation. */ + rule: string | null; + /** The full recommendation, used to record outcomes. */ + recommendation: { id: string; userId: string; confidence: number } | null; +} + +export interface UseEmotionDetectionOptions { + /** Master switch for automatic adaptation (default true). */ + adaptationEnabled?: boolean; + /** Whether the learner has granted engagement-tracking consent. */ + consent?: boolean; + /** Called whenever an adaptation is applied. */ + onAdaptationApplied?: (adaptation: AdaptiveState) => void; +} + +/** + * Emotion/engagement detection hook wired into the engagement-aware + * content adaptation engine (issue #408). + * + * Privacy: the camera stream never leaves the device. Only derived + * scores are reported to the backend, and only after the learner grants + * engagement-tracking consent via `setConsent(true)`. + */ +export const useEmotionDetection = (options: UseEmotionDetectionOptions = {}) => { const videoRef = useRef(null); + const streamRef = useRef(null); + const intervalRef = useRef | null>(null); const [state, setState] = useState({ dominantEmotion: 'neutral', engagementScore: 50, frustrationScore: 0, isDetecting: false, }); - const streamRef = useRef(null); + const [adaptive, setAdaptive] = useState({ + playbackRate: 1, + currentAction: null, + explanation: null, + isAdapting: false, + rule: null, + recommendation: null, + }); + const [preferences, setPreferences] = useState({ + ...DEFAULT_PREFERENCES, + adaptationEnabled: options.adaptationEnabled ?? true, + }); + const [consent, setConsentState] = useState({ + ...DEFAULT_CONSENT, + engagementTrackingConsent: options.consent ?? false, + }); + const optionsRef = useRef(options); + optionsRef.current = options; + + /** Report a derived signal to the backend — only with consent, fail-open. */ + const reportSignal = useCallback( + (signal: { engagementScore: number; frustrationScore: number; dominantEmotion: Emotion }) => { + if (!consent.engagementTrackingConsent) return; + const userId = typeof window !== 'undefined' + ? window.localStorage.getItem('learner_id') || 'anonymous' + : 'anonymous'; + adaptationApi.reportSignal(userId, signal).catch(() => { + // Fail open: the player keeps working un-adapted when offline/unreachable. + }); + }, + [consent.engagementTrackingConsent], + ); + + /** Evaluate the latest signal against the adaptation engine. */ + const evaluateAdaptation = useCallback( + (signal: { engagementScore: number; frustrationScore: number; dominantEmotion: Emotion }) => { + const prefs = preferencesRef.current; + const decision = decideAdaptation(signal, prefs); + const action = decision.action; + const isActive = action.kind !== 'none'; + + setAdaptive((prev) => { + const next: AdaptiveState = { + playbackRate: isActive ? playbackRateFor(action) : 1, + currentAction: isActive ? action : null, + explanation: isActive ? decision.explanation : null, + isAdapting: isActive, + rule: isActive ? decision.rule : null, + recommendation: isActive + ? { id: `local_${Date.now()}`, userId: 'local', confidence: decision.confidence } + : null, + }; + optionsRef.current.onAdaptationApplied?.(next); + return next; + }); + }, + [], + ); + + // Keep a ref so the interval callback always reads fresh preferences. + const preferencesRef = useRef(preferences); + preferencesRef.current = preferences; const startDetection = useCallback(async () => { try { @@ -29,38 +137,112 @@ export const useEmotionDetection = () => { videoRef.current.srcObject = stream; } streamRef.current = stream; - setState(prev => ({ ...prev, isDetecting: true, error: undefined })); - - // TensorFlow.js model integration point (simulation for demonstration logic) + setState((prev) => ({ ...prev, isDetecting: true, error: undefined })); + + // TensorFlow.js model integration point (simulation for demonstration logic). + // A real integration runs inference on videoRef.current frames here. + void tf; + const detectionInterval = setInterval(() => { if (!videoRef.current || !streamRef.current) return; - + const emotions: Emotion[] = ['neutral', 'happy', 'sad', 'angry', 'surprised']; const detectedEmotion = emotions[Math.floor(Math.random() * emotions.length)]; - + const isFrustrated = detectedEmotion === 'angry' || detectedEmotion === 'sad'; - const isEngaged = detectedEmotion === 'happy' || detectedEmotion === 'surprised' || detectedEmotion === 'neutral'; - - setState(prev => ({ - dominantEmotion: detectedEmotion, - engagementScore: Math.min(100, Math.max(0, prev.engagementScore + (isEngaged ? 5 : -5))), - frustrationScore: Math.min(100, Math.max(0, prev.frustrationScore + (isFrustrated ? 8 : -3))), - isDetecting: true - })); - }, 2000); + const isEngaged = + detectedEmotion === 'happy' || detectedEmotion === 'surprised' || detectedEmotion === 'neutral'; - return () => clearInterval(detectionInterval); + setState((prev) => { + const next = { + dominantEmotion: detectedEmotion, + engagementScore: Math.min(100, Math.max(0, prev.engagementScore + (isEngaged ? 5 : -5))), + frustrationScore: Math.min(100, Math.max(0, prev.frustrationScore + (isFrustrated ? 8 : -3))), + isDetecting: true, + }; + evaluateAdaptation(next); + reportSignal(next); + return next; + }); + }, 2000); + intervalRef.current = detectionInterval; } catch (err) { - setState(prev => ({ ...prev, error: 'Camera access denied or unavailable. Privacy settings may be blocking access.', isDetecting: false })); + setState((prev) => ({ + ...prev, + error: + 'Camera access denied or unavailable. Privacy settings may be blocking access.', + isDetecting: false, + })); } - }, []); + }, [evaluateAdaptation, reportSignal]); const stopDetection = useCallback(() => { + if (intervalRef.current) { + clearInterval(intervalRef.current); + intervalRef.current = null; + } if (streamRef.current) { - streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current = null; } - setState(prev => ({ ...prev, isDetecting: false, engagementScore: 50, frustrationScore: 0 })); + setState((prev) => ({ ...prev, isDetecting: false, engagementScore: 50, frustrationScore: 0 })); + setAdaptive({ + playbackRate: 1, + currentAction: null, + explanation: null, + isAdapting: false, + rule: null, + recommendation: null, + }); + }, []); + + /** Update adaptation preferences (user-overridable). */ + const updatePreference = useCallback((partial: Partial) => { + setPreferences((prev) => { + const next = { ...prev, ...partial }; + adaptationApi.updatePreferences(partial).catch(() => { + // Fail open — local preference still applies. + }); + return next; + }); + }, []); + + /** Update consent state; revocation also stops backend reporting. */ + const setConsent = useCallback((partial: Partial) => { + setConsentState((prev) => { + const next = { ...prev, ...partial }; + adaptationApi.updateConsent(partial).catch(() => { + // Fail open. + }); + return next; + }); + }, []); + + /** Dismiss the current adaptation and restore normal pacing. */ + const dismissAdaptation = useCallback(() => { + setAdaptive((prev) => { + if (prev.recommendation) { + adaptationApi.recordOutcome({ recommendationId: prev.recommendation.id, accepted: false }).catch(() => {}); + } + return { + playbackRate: 1, + currentAction: null, + explanation: null, + isAdapting: false, + rule: null, + recommendation: null, + }; + }); + }, []); + + /** Accept the current adaptation and keep the applied pacing. */ + const acceptAdaptation = useCallback(() => { + setAdaptive((prev) => { + if (prev.recommendation) { + adaptationApi.recordOutcome({ recommendationId: prev.recommendation.id, accepted: true }).catch(() => {}); + } + return { ...prev, isAdapting: false }; + }); }, []); useEffect(() => { @@ -69,5 +251,22 @@ export const useEmotionDetection = () => { }; }, [stopDetection]); - return { videoRef, ...state, startDetection, stopDetection }; -}; \ No newline at end of file + return { + videoRef, + ...state, + startDetection, + stopDetection, + // Engagement-aware adaptation (issue #408) + playbackRate: adaptive.playbackRate, + currentAdaptation: adaptive.currentAction, + adaptationExplanation: adaptive.explanation, + isAdapting: adaptive.isAdapting, + adaptationRule: adaptive.rule, + preferences, + consent, + updatePreference, + setConsent, + dismissAdaptation, + acceptAdaptation, + }; +}; diff --git a/frontend/src/lib/adaptationEngine.ts b/frontend/src/lib/adaptationEngine.ts new file mode 100644 index 00000000..4700ad17 --- /dev/null +++ b/frontend/src/lib/adaptationEngine.ts @@ -0,0 +1,229 @@ +/** + * Engagement-aware content adaptation (issue #408) + * + * Pure decision logic + thin API client for the adaptation endpoints. + * The decision rules mirror the backend `AdaptationService` so playback + * can adapt instantly on-device while the backend records signals, + * preferences, consent, and effectiveness. + * + * Privacy: only *derived* scores (engagement / frustration / emotion) + * are ever sent to the API, and only when the learner has granted + * engagement-tracking consent. Raw video never leaves the client. + */ + +export type Emotion = 'neutral' | 'happy' | 'sad' | 'angry' | 'fearful' | 'disgusted' | 'surprised'; +export type AdaptationKind = 'slow_down' | 'speed_up' | 'offer_review' | 'offer_hint' | 'simplify' | 'none'; + +export interface EngagementSignal { + engagementScore: number; + frustrationScore: number; + dominantEmotion: Emotion; + progress?: number; +} + +export interface AdaptationPreference { + adaptationEnabled: boolean; + pacingEnabled: boolean; + difficultyEnabled: boolean; + hintsEnabled: boolean; + maxPlaybackRate: number; + minPlaybackRate: number; +} + +export interface ConsentState { + engagementTrackingConsent: boolean; + biometricConsent: boolean; +} + +export interface AdaptationAction { + kind: AdaptationKind; + playbackRate?: number; + difficulty?: string; + message?: string; +} + +export interface AdaptationRecommendation { + id: string; + userId: string; + action: AdaptationAction; + explanation: string; + rule: string; + confidence: number; + overridable: boolean; + createdAt: number; +} + +export const DEFAULT_PREFERENCES: AdaptationPreference = { + adaptationEnabled: true, + pacingEnabled: true, + difficultyEnabled: true, + hintsEnabled: true, + maxPlaybackRate: 1.5, + minPlaybackRate: 0.5, +}; + +export const DEFAULT_CONSENT: ConsentState = { + engagementTrackingConsent: false, + biometricConsent: false, +}; + +const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); + +/** + * Pure, deterministic adaptation decision. Mirrors the backend rule set + * so on-device playback and server-side history stay consistent. + */ +export function decideAdaptation( + signal: EngagementSignal, + prefs: AdaptationPreference = DEFAULT_PREFERENCES, +): { action: AdaptationAction; rule: string; explanation: string; confidence: number } { + const engagement = clamp(Number(signal.engagementScore) || 0, 0, 100); + const frustration = clamp(Number(signal.frustrationScore) || 0, 0, 100); + + if (!prefs.adaptationEnabled) { + return { + action: { kind: 'none', message: 'Automatic adaptation is disabled in your preferences.' }, + rule: 'preference-disabled', + explanation: 'Adaptation is turned off in your preferences, so no changes were made.', + confidence: 0, + }; + } + + if (prefs.hintsEnabled && frustration > 75) { + return { + action: { + kind: 'offer_review', + message: 'You seem frustrated. Would you like to review the previous section or take a short break?', + }, + rule: 'frustration-review', + explanation: + `Frustration is at ${Math.round(frustration)}/100, above the 75 threshold. ` + + 'Reviewing earlier material reduces cognitive load.', + confidence: 0.9, + }; + } + + if (prefs.pacingEnabled && frustration > 60) { + const rate = Math.max(prefs.minPlaybackRate, 0.75); + return { + action: { kind: 'slow_down', playbackRate: rate }, + rule: 'frustration-slow-down', + explanation: + `Frustration is at ${Math.round(frustration)}/100, above the 60 threshold. ` + + `Slowing the pacing to ${rate}× gives you more time to process.`, + confidence: 0.85, + }; + } + + if (prefs.hintsEnabled && engagement < 30) { + return { + action: { kind: 'offer_hint', message: 'Engagement is dropping — here is a hint to help you stay on track.' }, + rule: 'low-engagement-hint', + explanation: + `Engagement is at ${Math.round(engagement)}/100, below the 30 threshold. ` + + 'An alternate explanation or hint is offered to re-engage you.', + confidence: 0.8, + }; + } + + if (prefs.difficultyEnabled && engagement < 40) { + return { + action: { kind: 'simplify', difficulty: 'beginner' }, + rule: 'low-engagement-simplify', + explanation: + `Engagement is at ${Math.round(engagement)}/100, below the 40 threshold. ` + + 'The content difficulty is temporarily simplified to rebuild confidence.', + confidence: 0.75, + }; + } + + if (prefs.pacingEnabled && engagement > 80 && frustration < 40) { + const rate = Math.min(prefs.maxPlaybackRate, 1.25); + return { + action: { kind: 'speed_up', playbackRate: rate }, + rule: 'high-engagement-speed-up', + explanation: + `Engagement is at ${Math.round(engagement)}/100 with low frustration, above the 80 threshold. ` + + `Pacing increased to ${rate}× to match your flow state.`, + confidence: 0.85, + }; + } + + return { + action: { kind: 'none', message: 'No adaptation needed right now.' }, + rule: 'standard-pacing', + explanation: + `Engagement ${Math.round(engagement)}/100 and frustration ${Math.round(frustration)}/100 are both ` + + 'within the healthy band, so the current pacing is kept.', + confidence: 0.7, + }; +} + +/** + * Returns the playback rate implied by a decision (1 = unchanged). + */ +export function playbackRateFor(action: AdaptationAction): number { + if (action.kind === 'slow_down' && action.playbackRate !== undefined) return action.playbackRate; + if (action.kind === 'speed_up' && action.playbackRate !== undefined) return action.playbackRate; + return 1; +} + +const API_BASE = '/api/adaptation'; + +const authHeaders = (): Record => { + const token = + typeof window !== 'undefined' ? window.localStorage.getItem('admin_token') : null; + return token ? { Authorization: `Bearer ${token}` } : {}; +}; + +async function request(path: string, init?: RequestInit): Promise { + const res = await fetch(`${API_BASE}${path}`, { + ...init, + headers: { 'Content-Type': 'application/json', ...authHeaders(), ...(init?.headers || {}) }, + }); + if (!res.ok) { + throw new Error(`Adaptation API request failed (${res.status})`); + } + return res.json(); +} + +/** + * Thin client for the adaptation endpoints. All calls fail open — the + * player should keep working (un-adapted) when the API is unreachable. + */ +export const adaptationApi = { + reportSignal(userId: string, signal: EngagementSignal): Promise<{ stored: boolean }> { + return request<{ stored: boolean }>('/signal', { + method: 'POST', + body: JSON.stringify({ ...signal, userId }), + }); + }, + getRecommendation(userId: string, signal: EngagementSignal): Promise<{ data: AdaptationRecommendation }> { + return request<{ data: AdaptationRecommendation }>('/recommend', { + method: 'POST', + body: JSON.stringify({ ...signal, userId }), + }); + }, + updatePreferences(prefs: Partial): Promise<{ data: AdaptationPreference }> { + return request<{ data: AdaptationPreference }>('/preferences', { + method: 'PUT', + body: JSON.stringify(prefs), + }); + }, + updateConsent(consent: Partial): Promise<{ data: ConsentState }> { + return request<{ data: ConsentState }>('/consent', { + method: 'PUT', + body: JSON.stringify(consent), + }); + }, + recordOutcome(outcome: { + recommendationId: string; + accepted: boolean; + quizScoreAfter?: number; + }): Promise<{ data: unknown }> { + return request<{ data: unknown }>('/outcome', { + method: 'POST', + body: JSON.stringify(outcome), + }); + }, +};