diff --git a/crates/pulse-common/src/types.rs b/crates/pulse-common/src/types.rs index 275c343..b26c449 100644 --- a/crates/pulse-common/src/types.rs +++ b/crates/pulse-common/src/types.rs @@ -103,7 +103,14 @@ pub struct EventPayload { pub struct IdentifyPayload { #[serde(default)] pub user_id: Option, + /// Person properties to set (overwrites existing values). pub traits: serde_json::Value, + /// Person properties set only if not already present (`$set_once`). + #[serde(default)] + pub set_once: Option, + /// Person properties to remove (`$unset`). + #[serde(default)] + pub unset: Option>, #[serde(default)] pub account_id: Option, #[serde(default)] diff --git a/crates/pulse-server/src/main.rs b/crates/pulse-server/src/main.rs index 7a7737c..e9e35b2 100644 --- a/crates/pulse-server/src/main.rs +++ b/crates/pulse-server/src/main.rs @@ -114,6 +114,11 @@ async fn main() -> anyhow::Result<()> { // Scheduled alert evaluation (fires drop/outage alerts without traffic) services::alerts::start_alert_evaluator_task(state.clone()); + // Person merges: make sure ingest can follow redirects even after a Redis flush + if let Err(e) = services::persons::warm_redirects(&state).await { + tracing::warn!("Failed to warm person redirects: {e}"); + } + // Recalculate stale dynamic person cohorts services::person_cohorts::start_recalc_task(state.clone()); info!("Email report scheduler started"); @@ -234,6 +239,31 @@ async fn main() -> anyhow::Result<()> { "/api/v1/saved-insights/{id}/versions/{version}/restore", axum::routing::post(routes::saved_insights::restore), ) + // Persons + .route( + "/api/v2/persons/query", + axum::routing::post(routes::persons::query), + ) + .route( + "/api/v2/persons/{visitor_id}", + get(routes::persons::get).delete(routes::persons::delete), + ) + .route( + "/api/v2/persons/{visitor_id}/events", + get(routes::persons::events), + ) + .route( + "/api/v2/persons/{visitor_id}/sessions", + get(routes::persons::sessions), + ) + .route( + "/api/v2/persons/{visitor_id}/properties", + axum::routing::patch(routes::persons::update_properties), + ) + .route( + "/api/v2/persons/{visitor_id}/merge", + axum::routing::post(routes::persons::merge), + ) // Person cohorts (v1 /api/v1/cohorts remains the acquisition cohort matrix) .route( "/api/v2/cohorts", diff --git a/crates/pulse-server/src/routes/ingest.rs b/crates/pulse-server/src/routes/ingest.rs index a42187f..39acf15 100644 --- a/crates/pulse-server/src/routes/ingest.rs +++ b/crates/pulse-server/src/routes/ingest.rs @@ -201,6 +201,12 @@ async fn process_collect_envelope( }); } + // Events from a merged-away visitor id belong to the person it was merged into. + let mut envelope = envelope; + envelope.visitor_id = + crate::services::persons::resolve_visitor(state, auth.project_id, &envelope.visitor_id) + .await; + let now = envelope .timestamp .map(|ts| chrono::DateTime::from_timestamp_millis(ts).unwrap_or_else(Utc::now)) @@ -304,6 +310,20 @@ async fn process_collect_envelope( .await .map_err(|e| AppError::Internal(e.to_string()))?; + // `$set` / `$set_once` / `$unset` in event properties update the person. + if let Some(update) = identity::PersonUpdate::from_event_data(payload.data.as_ref()) { + identity::apply_person_update( + &state.db, + auth.project_id, + &envelope.visitor_id, + None, + &update, + now, + false, + ) + .await?; + } + let event = BufferedEvent { project_id: auth.project_id, session_id, @@ -379,6 +399,8 @@ async fn process_collect_envelope( &envelope.visitor_id, payload.user_id.as_deref(), &payload.traits, + payload.set_once.as_ref(), + payload.unset.as_deref().unwrap_or_default(), payload.account_id.as_deref(), payload.account_name.as_deref(), payload.account_traits.as_ref(), diff --git a/crates/pulse-server/src/routes/mod.rs b/crates/pulse-server/src/routes/mod.rs index d43b9ec..180b4b9 100644 --- a/crates/pulse-server/src/routes/mod.rs +++ b/crates/pulse-server/src/routes/mod.rs @@ -8,6 +8,7 @@ pub mod features_ext; pub mod health; pub mod ingest; pub mod insights; +pub mod persons; pub mod query; pub mod saved_insights; pub mod script; diff --git a/crates/pulse-server/src/routes/persons.rs b/crates/pulse-server/src/routes/persons.rs new file mode 100644 index 0000000..9f038a0 --- /dev/null +++ b/crates/pulse-server/src/routes/persons.rs @@ -0,0 +1,153 @@ +use axum::extract::{Path, Query}; +use axum::response::IntoResponse; +use axum::{Extension, Json}; +use serde::Deserialize; + +use crate::error::AppResult; +use crate::middleware::auth::AuthenticatedProject; +use crate::services; +use crate::services::persons::{self, EventsQuery, PersonsQuery, PropertiesPatch}; +use crate::state::SharedState; + +/// People data is governed by the `identity` module. +const MODULE: &str = "identity"; + +async fn can_read(state: &SharedState, auth: &AuthenticatedProject) -> AppResult<()> { + auth.require_scope("query")?; + services::modules::require_module_read(state, auth.project_id, MODULE, &auth.allowed_modules) + .await +} + +async fn can_write(state: &SharedState, auth: &AuthenticatedProject) -> AppResult<()> { + auth.require_scope("query")?; + services::modules::require_module_write(state, auth.project_id, MODULE, &auth.allowed_modules) + .await +} + +// POST /api/v2/persons/query +pub async fn query( + Extension(state): Extension, + auth: AuthenticatedProject, + Json(q): Json, +) -> AppResult { + can_read(&state, &auth).await?; + if q.uses_cohorts() { + services::modules::require_module_read( + &state, + auth.project_id, + "cohorts", + &auth.allowed_modules, + ) + .await?; + } + Ok(Json(persons::list(&state.db, auth.project_id, &q).await?)) +} + +// GET /api/v2/persons/{visitor_id} +pub async fn get( + Extension(state): Extension, + auth: AuthenticatedProject, + Path(visitor_id): Path, +) -> AppResult { + can_read(&state, &auth).await?; + // Cohort memberships are cohorts-module data: include them only if readable. + let include_cohorts = services::modules::require_module_read( + &state, + auth.project_id, + "cohorts", + &auth.allowed_modules, + ) + .await + .is_ok(); + Ok(Json( + persons::get(&state.db, auth.project_id, &visitor_id, include_cohorts).await?, + )) +} + +// GET /api/v2/persons/{visitor_id}/events +pub async fn events( + Extension(state): Extension, + auth: AuthenticatedProject, + Path(visitor_id): Path, + Query(q): Query, +) -> AppResult { + can_read(&state, &auth).await?; + Ok(Json( + persons::events(&state.db, auth.project_id, &visitor_id, &q).await?, + )) +} + +#[derive(Debug, Deserialize)] +pub struct Page { + #[serde(default = "default_limit")] + pub limit: i64, + #[serde(default)] + pub offset: i64, +} + +fn default_limit() -> i64 { + 100 +} + +// GET /api/v2/persons/{visitor_id}/sessions +pub async fn sessions( + Extension(state): Extension, + auth: AuthenticatedProject, + Path(visitor_id): Path, + Query(p): Query, +) -> AppResult { + can_read(&state, &auth).await?; + Ok(Json( + persons::sessions(&state.db, auth.project_id, &visitor_id, p.limit, p.offset).await?, + )) +} + +// PATCH /api/v2/persons/{visitor_id}/properties +pub async fn update_properties( + Extension(state): Extension, + auth: AuthenticatedProject, + Path(visitor_id): Path, + Json(p): Json, +) -> AppResult { + can_write(&state, &auth).await?; + Ok(Json( + persons::update_properties(&state.db, auth.project_id, &visitor_id, &p).await?, + )) +} + +#[derive(Debug, Deserialize)] +pub struct MergeBody { + pub visitor_ids: Vec, +} + +// POST /api/v2/persons/{visitor_id}/merge +pub async fn merge( + Extension(state): Extension, + auth: AuthenticatedProject, + Path(visitor_id): Path, + Json(b): Json, +) -> AppResult { + can_write(&state, &auth).await?; + Ok(Json( + persons::merge(&state, auth.project_id, &visitor_id, &b.visitor_ids).await?, + )) +} + +#[derive(Debug, Deserialize, Default)] +pub struct DeleteParams { + #[serde(default)] + pub delete_events: bool, +} + +// DELETE /api/v2/persons/{visitor_id} +pub async fn delete( + Extension(state): Extension, + auth: AuthenticatedProject, + Path(visitor_id): Path, + Query(p): Query, +) -> AppResult { + can_write(&state, &auth).await?; + Ok(Json( + persons::delete(&state.db, auth.project_id, &visitor_id, p.delete_events).await?, + )) +} diff --git a/crates/pulse-server/src/services/identity.rs b/crates/pulse-server/src/services/identity.rs index 635053e..5d1b7bf 100644 --- a/crates/pulse-server/src/services/identity.rs +++ b/crates/pulse-server/src/services/identity.rs @@ -179,27 +179,63 @@ fn empty_array() -> serde_json::Value { serde_json::json!([]) } -/// Persist an identify call and merge traits into the visitor's profile. -pub async fn identify_user( +/// Person property changes (PostHog `$set` / `$set_once` / `$unset`). +#[derive(Debug, Clone, Default)] +pub struct PersonUpdate { + /// Overwrite these keys. + pub set: serde_json::Value, + /// Set these keys only if the person does not have them yet. + pub set_once: serde_json::Value, + /// Remove these keys. + pub unset: Vec, +} + +impl PersonUpdate { + /// Extract `$set`, `$set_once` and `$unset` from an event's properties. + pub fn from_event_data(data: Option<&serde_json::Value>) -> Option { + let obj = data?.as_object()?; + let pick = |k: &str| obj.get(k).filter(|v| v.is_object()).cloned(); + let set = pick("$set"); + let set_once = pick("$set_once"); + let unset: Vec = obj + .get("$unset") + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + if set.is_none() && set_once.is_none() && unset.is_empty() { + return None; + } + Some(Self { + set: set.unwrap_or_else(empty_object), + set_once: set_once.unwrap_or_else(empty_object), + unset, + }) + } +} + +/// Create or update a person: `traits = ((set_once || traits) || set) - unset`. +pub async fn apply_person_update( db: &PgPool, project_id: Uuid, visitor_id: &str, user_id: Option<&str>, - traits: &serde_json::Value, - account_id: Option<&str>, - account_name: Option<&str>, - account_traits: Option<&serde_json::Value>, - account_role: Option<&str>, - identified_at: DateTime, + update: &PersonUpdate, + seen_at: DateTime, + identified: bool, ) -> AppResult { - let user_id = normalize_optional(user_id); - let profile: UserProfile = sqlx::query_as(&format!( + let set = normalized_json_object(Some(&update.set), "traits")?; + let set_once = normalized_json_object(Some(&update.set_once), "set_once")?; + Ok(sqlx::query_as(&format!( "INSERT INTO user_profiles \ (project_id, visitor_id, user_id, traits, first_seen_at, last_seen_at, identified_at) \ - VALUES ($1, $2, $3, $4, $5, $5, $5) \ + VALUES ($1, $2, $3, ($6::jsonb || $4::jsonb) - $7::text[], $5, $5, CASE WHEN $8 THEN $5 END) \ ON CONFLICT (project_id, visitor_id) DO UPDATE SET \ user_id = COALESCE(EXCLUDED.user_id, user_profiles.user_id), \ - traits = user_profiles.traits || EXCLUDED.traits, \ + traits = (($6::jsonb || user_profiles.traits) || $4::jsonb) - $7::text[], \ first_seen_at = LEAST(user_profiles.first_seen_at, EXCLUDED.first_seen_at), \ last_seen_at = GREATEST(user_profiles.last_seen_at, EXCLUDED.last_seen_at), \ identified_at = COALESCE(user_profiles.identified_at, EXCLUDED.identified_at), \ @@ -208,10 +244,46 @@ pub async fn identify_user( )) .bind(project_id) .bind(visitor_id) - .bind(user_id) - .bind(traits) - .bind(identified_at) + .bind(normalize_optional(user_id)) + .bind(&set) + .bind(seen_at) + .bind(&set_once) + .bind(&update.unset) + .bind(identified) .fetch_one(db) + .await?) +} + +/// Persist an identify call and merge traits into the visitor's profile. +#[allow(clippy::too_many_arguments)] +pub async fn identify_user( + db: &PgPool, + project_id: Uuid, + visitor_id: &str, + user_id: Option<&str>, + traits: &serde_json::Value, + set_once: Option<&serde_json::Value>, + unset: &[String], + account_id: Option<&str>, + account_name: Option<&str>, + account_traits: Option<&serde_json::Value>, + account_role: Option<&str>, + identified_at: DateTime, +) -> AppResult { + let user_id = normalize_optional(user_id); + let profile = apply_person_update( + db, + project_id, + visitor_id, + user_id, + &PersonUpdate { + set: traits.clone(), + set_once: set_once.cloned().unwrap_or_else(empty_object), + unset: unset.to_vec(), + }, + identified_at, + true, + ) .await?; if let Some(user_id) = user_id { @@ -956,6 +1028,8 @@ async fn sync_scim_user_to_identity(db: &PgPool, user: &ScimUser) -> AppResult) -> AppError { + AppError::BadRequest(msg.into()) +} + +fn redirect_key(state: &SharedState, project_id: Uuid) -> String { + state.redis_key(&format!("person_redirects:{project_id}")) +} + +/// The visitor id an incoming event should be recorded under (follows merges). +/// Redis errors fall back to the original id so ingestion never fails here. +pub async fn resolve_visitor(state: &SharedState, project_id: Uuid, visitor_id: &str) -> String { + let mut redis = state.redis.clone(); + match redis + .hget::<_, _, Option>(redirect_key(state, project_id), visitor_id) + .await + { + Ok(Some(target)) => target, + _ => visitor_id.to_string(), + } +} + +/// Load all redirects into Redis (startup; Redis may have been flushed). +pub async fn warm_redirects(state: &SharedState) -> AppResult<()> { + let rows: Vec<(Uuid, String, String)> = + sqlx::query_as("SELECT project_id, from_visitor_id, to_visitor_id FROM person_redirects") + .fetch_all(&state.db) + .await?; + let mut redis = state.redis.clone(); + for (project_id, from, to) in rows { + let _: () = redis + .hset(redirect_key(state, project_id), from, to) + .await?; + } + Ok(()) +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct PersonsQuery { + /// Matches visitor id, user id or any property value (case-insensitive). + #[serde(default)] + pub search: Option, + /// Person-property and cohort filters (ANDed). + #[serde(default)] + pub properties: Vec, + /// `last_seen` (default), `first_seen` or `identified`. + #[serde(default)] + pub order: Option, + #[serde(default = "default_limit")] + pub limit: i64, + #[serde(default)] + pub offset: i64, +} + +fn default_limit() -> i64 { + 100 +} + +impl PersonsQuery { + pub fn uses_cohorts(&self) -> bool { + self.properties + .iter() + .any(|f| f.property_type == PropertyType::Cohort) + } +} + +pub async fn list(db: &PgPool, project_id: Uuid, q: &PersonsQuery) -> AppResult { + if !(1..=1000).contains(&q.limit) || q.offset < 0 { + return Err(bad("limit must be 1-1000 and offset >= 0")); + } + let order = match q.order.as_deref().unwrap_or("last_seen") { + "last_seen" => "up.last_seen_at DESC", + "first_seen" => "up.first_seen_at DESC", + "identified" => "up.identified_at DESC NULLS LAST", + other => return Err(bad(format!("unknown order '{other}'"))), + }; + // `ev` exposes project_id/visitor_id so cohort filters work on persons too. + let mut qb = QueryBuilder::::new( + "SELECT up.visitor_id, up.user_id, up.traits, up.first_seen_at, up.last_seen_at, \ + up.identified_at, COUNT(*) OVER () AS total \ + FROM user_profiles up \ + CROSS JOIN LATERAL (SELECT up.project_id AS project_id, up.visitor_id AS visitor_id) ev \ + WHERE up.project_id = ", + ); + qb.push_bind(project_id); + if let Some(s) = q.search.as_deref().filter(|s| !s.trim().is_empty()) { + let like = format!( + "%{}%", + s.trim() + .replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") + ); + qb.push(" AND (up.visitor_id ILIKE "); + qb.push_bind(like.clone()); + qb.push(" OR up.user_id ILIKE "); + qb.push_bind(like.clone()); + qb.push(" OR up.traits::text ILIKE "); + qb.push_bind(like); + qb.push(")"); + } + for f in &q.properties { + if !matches!(f.property_type, PropertyType::Person | PropertyType::Cohort) { + return Err(bad( + "persons can be filtered by person properties and cohorts only", + )); + } + qb.push(" AND "); + push_filter(&mut qb, f)?; + } + qb.push(format!(" ORDER BY {order}, up.visitor_id LIMIT ")); + qb.push_bind(q.limit); + qb.push(" OFFSET "); + qb.push_bind(q.offset); + type Row = ( + String, + Option, + Value, + DateTime, + DateTime, + Option>, + i64, + ); + let rows: Vec = qb + .build_query_as() + .fetch_all(db) + .await + .map_err(map_query_error)?; + let total = rows.first().map(|r| r.6).unwrap_or(0); + let data: Vec = rows + .into_iter() + .map(|(v, u, t, first, last, ident, _)| { + json!({ "visitor_id": v, "user_id": u, "traits": t, "first_seen_at": first, + "last_seen_at": last, "identified_at": ident }) + }) + .collect(); + Ok(json!({ "data": data, "total": total, "limit": q.limit, "offset": q.offset })) +} + +async fn exists(db: &PgPool, project_id: Uuid, visitor_id: &str) -> AppResult { + let (n,): (i64,) = sqlx::query_as( + "SELECT (SELECT COUNT(*) FROM user_profiles WHERE project_id = $1 AND visitor_id = $2) \ + + (SELECT COUNT(*) FROM (SELECT 1 FROM sessions WHERE project_id = $1 AND visitor_id = $2 LIMIT 1) s)", + ) + .bind(project_id) + .bind(visitor_id) + .fetch_one(db) + .await?; + Ok(n > 0) +} + +async fn require_person(db: &PgPool, project_id: Uuid, visitor_id: &str) -> AppResult<()> { + if exists(db, project_id, visitor_id).await? { + Ok(()) + } else { + Err(AppError::NotFound("Person not found".into())) + } +} + +/// `include_cohorts` is false when the caller may not read the cohorts module. +pub async fn get( + db: &PgPool, + project_id: Uuid, + visitor_id: &str, + include_cohorts: bool, +) -> AppResult { + require_person(db, project_id, visitor_id).await?; + let profile: Option<( + Option, + Value, + DateTime, + DateTime, + Option>, + )> = sqlx::query_as( + "SELECT user_id, traits, first_seen_at, last_seen_at, identified_at FROM user_profiles \ + WHERE project_id = $1 AND visitor_id = $2", + ) + .bind(project_id) + .bind(visitor_id) + .fetch_optional(db) + .await?; + let user_id = profile.as_ref().and_then(|p| p.0.clone()); + let aliases: Vec<(String,)> = sqlx::query_as( + "SELECT DISTINCT visitor_id FROM user_aliases WHERE project_id = $1 AND user_id = $2 ORDER BY 1", + ) + .bind(project_id) + .bind(&user_id) + .fetch_all(db) + .await?; + let stats: (i64, i64, i64, Option>, Option>) = sqlx::query_as( + "SELECT (SELECT COUNT(*) FROM pageviews WHERE project_id = $1 AND visitor_id = $2), \ + (SELECT COUNT(*) FROM events WHERE project_id = $1 AND visitor_id = $2), \ + COUNT(*), MIN(first_at), MAX(last_at) FROM sessions WHERE project_id = $1 AND visitor_id = $2", + ) + .bind(project_id) + .bind(visitor_id) + .fetch_one(db) + .await?; + let cohorts: Vec<(Uuid, String, bool)> = if include_cohorts { + sqlx::query_as( + "SELECT c.id, c.name, c.is_static FROM cohort_people cp JOIN person_cohorts c ON c.id = cp.cohort_id \ + WHERE c.project_id = $1 AND cp.visitor_id = $2 AND NOT c.deleted ORDER BY c.name", + ) + .bind(project_id) + .bind(visitor_id) + .fetch_all(db) + .await? + } else { + Vec::new() + }; + let merged_from: Vec<(String,)> = sqlx::query_as( + "SELECT from_visitor_id FROM person_redirects WHERE project_id = $1 AND to_visitor_id = $2 ORDER BY 1", + ) + .bind(project_id) + .bind(visitor_id) + .fetch_all(db) + .await?; + Ok(json!({ + "visitor_id": visitor_id, + "user_id": user_id, + "traits": profile.as_ref().map(|p| p.1.clone()).unwrap_or_else(|| json!({})), + "first_seen_at": profile.as_ref().map(|p| p.2).or(stats.3), + "last_seen_at": profile.as_ref().map(|p| p.3).or(stats.4), + "identified_at": profile.as_ref().and_then(|p| p.4), + "aliases": aliases.into_iter().map(|a| a.0).collect::>(), + "merged_from": merged_from.into_iter().map(|a| a.0).collect::>(), + "stats": { "pageviews": stats.0, "events": stats.1, "sessions": stats.2 }, + // Omitted (null) when the caller may not read cohorts. + "cohorts": include_cohorts.then(|| cohorts.into_iter().map(|(id, name, s)| json!({"id": id, "name": name, "is_static": s})).collect::>()), + })) +} + +#[derive(Debug, Clone, Deserialize)] +pub struct EventsQuery { + /// Only this event (`$pageview` for pageviews). + #[serde(default)] + pub event: Option, + #[serde(default = "default_limit")] + pub limit: i64, + #[serde(default)] + pub offset: i64, +} + +/// A person's events (pageviews and custom events), newest first. +pub async fn events( + db: &PgPool, + project_id: Uuid, + visitor_id: &str, + q: &EventsQuery, +) -> AppResult { + require_person(db, project_id, visitor_id).await?; + if !(1..=1000).contains(&q.limit) || q.offset < 0 { + return Err(bad("limit must be 1-1000 and offset >= 0")); + } + let names: Option> = q.event.clone().map(|e| vec![e]); + let mut qb = QueryBuilder::::new( + "SELECT ev.event, ev.created_at, ev.session_id, ev.properties", + ); + push_event_source( + &mut qb, + &EventSource { + project_id, + from: DateTime::::UNIX_EPOCH, + to: Utc::now() + chrono::Duration::days(1), + events: names.as_deref(), + joins: Default::default(), + }, + ); + qb.push(" WHERE ev.visitor_id = "); + qb.push_bind(visitor_id.to_string()); + qb.push(" ORDER BY ev.created_at DESC LIMIT "); + qb.push_bind(q.limit); + qb.push(" OFFSET "); + qb.push_bind(q.offset); + let rows: Vec<(String, DateTime, Uuid, Value)> = qb + .build_query_as() + .fetch_all(db) + .await + .map_err(map_query_error)?; + let data: Vec = rows + .into_iter() + .map(|(e, at, s, p)| json!({ "event": e, "created_at": at, "session_id": s, "properties": p })) + .collect(); + Ok(json!({ "data": data, "limit": q.limit, "offset": q.offset })) +} + +/// A person's sessions, newest first, flagging those with a recording. +pub async fn sessions( + db: &PgPool, + project_id: Uuid, + visitor_id: &str, + limit: i64, + offset: i64, +) -> AppResult { + require_person(db, project_id, visitor_id).await?; + if !(1..=1000).contains(&limit) || offset < 0 { + return Err(bad("limit must be 1-1000 and offset >= 0")); + } + let rows: Vec<(Uuid, DateTime, DateTime, Option, Option, i32, i32, i64, bool, Option, Option, Option, Option, bool)> = sqlx::query_as( + "SELECT s.id, s.first_at, s.last_at, s.entry_page, s.exit_page, s.pageview_count, s.event_count, \ + s.duration_ms, s.is_bounce, s.browser, s.os, s.device, s.country, \ + EXISTS (SELECT 1 FROM session_recordings r WHERE r.project_id = s.project_id AND r.session_id = s.id) \ + FROM sessions s WHERE s.project_id = $1 AND s.visitor_id = $2 ORDER BY s.first_at DESC LIMIT $3 OFFSET $4", + ) + .bind(project_id) + .bind(visitor_id) + .bind(limit) + .bind(offset) + .fetch_all(db) + .await?; + let data: Vec = rows + .into_iter() + .map(|r| json!({ + "session_id": r.0, "first_at": r.1, "last_at": r.2, "entry_page": r.3, "exit_page": r.4, + "pageviews": r.5, "events": r.6, "duration_ms": r.7, "is_bounce": r.8, + "browser": r.9, "os": r.10, "device": r.11, "country": r.12, "has_recording": r.13, + })) + .collect(); + Ok(json!({ "data": data, "limit": limit, "offset": offset })) +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct PropertiesPatch { + #[serde(default)] + pub set: Option, + #[serde(default)] + pub set_once: Option, + #[serde(default)] + pub unset: Vec, +} + +pub async fn update_properties( + db: &PgPool, + project_id: Uuid, + visitor_id: &str, + p: &PropertiesPatch, +) -> AppResult { + require_person(db, project_id, visitor_id).await?; + let profile = identity::apply_person_update( + db, + project_id, + visitor_id, + None, + &PersonUpdate { + set: p.set.clone().unwrap_or_else(|| json!({})), + set_once: p.set_once.clone().unwrap_or_else(|| json!({})), + unset: p.unset.clone(), + }, + Utc::now(), + false, + ) + .await?; + Ok(json!({ "visitor_id": profile.visitor_id, "traits": profile.traits })) +} + +/// Merge `others` into `target`: their events, sessions and memberships move +/// to the target, their properties fill gaps in the target's (the target's +/// values win), and future events under their ids follow a redirect. +pub async fn merge( + state: &SharedState, + project_id: Uuid, + target: &str, + others: &[String], +) -> AppResult { + let db = &state.db; + if others.is_empty() || others.len() > MAX_MERGE { + return Err(bad(format!("merge 1-{MAX_MERGE} persons at a time"))); + } + if others.iter().any(|o| o == target) { + return Err(bad("cannot merge a person into themselves")); + } + require_person(db, project_id, target).await?; + for o in others { + require_person(db, project_id, o).await?; + } + let mut tx = db.begin().await?; + // Properties: others fill gaps (in the given order), the target wins. + let merged: (Value, Option, Option>) = sqlx::query_as( + // Later keys win in jsonb_object_agg, so aggregate in descending list + // order: the first-listed person's value wins among the others. + "SELECT COALESCE(jsonb_object_agg(k, v ORDER BY ord DESC) FILTER (WHERE k IS NOT NULL), '{}'::jsonb), \ + (array_agg(user_id ORDER BY ord) FILTER (WHERE user_id IS NOT NULL))[1], MIN(first_seen_at) \ + FROM (SELECT o.ord, up.user_id, up.first_seen_at, t.key AS k, t.value AS v \ + FROM unnest($2::varchar[]) WITH ORDINALITY AS o(visitor_id, ord) \ + JOIN user_profiles up ON up.project_id = $1 AND up.visitor_id = o.visitor_id \ + LEFT JOIN LATERAL jsonb_each(up.traits) t ON TRUE) x", + ) + .bind(project_id) + .bind(others) + .fetch_one(&mut *tx) + .await?; + sqlx::query( + "INSERT INTO user_profiles (project_id, visitor_id, user_id, traits, first_seen_at, last_seen_at) \ + VALUES ($1, $2, $3, $4, COALESCE($5, NOW()), NOW()) \ + ON CONFLICT (project_id, visitor_id) DO UPDATE SET \ + traits = $4::jsonb || user_profiles.traits, \ + user_id = COALESCE(user_profiles.user_id, EXCLUDED.user_id), \ + first_seen_at = LEAST(user_profiles.first_seen_at, EXCLUDED.first_seen_at), \ + updated_at = NOW()", + ) + .bind(project_id) + .bind(target) + .bind(&merged.1) + .bind(&merged.0) + .bind(merged.2) + .execute(&mut *tx) + .await?; + // Rows unique per (key, visitor_id) would collide once rewritten to the + // target: keep the target's row, else the first-listed source's. + for (table, key) in [ + ("experiment_exposures", "experiment_id"), + ("account_memberships", "account_id"), + ] { + sqlx::query(&format!( + "DELETE FROM {table} o WHERE o.project_id = $1 AND o.visitor_id = ANY($2) AND EXISTS ( \ + SELECT 1 FROM {table} k WHERE k.project_id = $1 AND k.{key} = o.{key} AND ( \ + k.visitor_id = $3 OR (k.visitor_id = ANY($2) \ + AND array_position($2, k.visitor_id) < array_position($2, o.visitor_id))))" + )) + .bind(project_id) + .bind(others) + .bind(target) + .execute(&mut *tx) + .await?; + } + sqlx::query( + "INSERT INTO user_aliases (project_id, user_id, visitor_id) \ + SELECT project_id, user_id, $3 FROM user_aliases WHERE project_id = $1 AND visitor_id = ANY($2) \ + ON CONFLICT DO NOTHING", + ) + .bind(project_id).bind(others).bind(target).execute(&mut *tx).await?; + sqlx::query( + "INSERT INTO cohort_people (cohort_id, visitor_id) \ + SELECT cp.cohort_id, $3 FROM cohort_people cp JOIN person_cohorts c ON c.id = cp.cohort_id \ + WHERE c.project_id = $1 AND cp.visitor_id = ANY($2) ON CONFLICT DO NOTHING", + ) + .bind(project_id).bind(others).bind(target).execute(&mut *tx).await?; + sqlx::query( + "DELETE FROM cohort_people cp USING person_cohorts c \ + WHERE cp.cohort_id = c.id AND c.project_id = $1 AND cp.visitor_id = ANY($2)", + ) + .bind(project_id) + .bind(others) + .execute(&mut *tx) + .await?; + let mut moved = serde_json::Map::new(); + for table in VISITOR_TABLES { + // Aliases and profiles were copied onto the target above; the rest move. + let delete = matches!(*table, "user_aliases" | "user_profiles"); + let sql = if delete { + format!("DELETE FROM {table} WHERE project_id = $1 AND visitor_id = ANY($2)") + } else { + format!( + "UPDATE {table} SET visitor_id = $3 WHERE project_id = $1 AND visitor_id = ANY($2)" + ) + }; + let q = sqlx::query(&sql).bind(project_id).bind(others); + let q = if delete { q } else { q.bind(target) }; + let n = q.execute(&mut *tx).await?.rows_affected(); + moved.insert(table.to_string(), json!(n)); + } + // Redirect the merged ids (and anything already redirected to them). + sqlx::query( + "UPDATE person_redirects SET to_visitor_id = $3 WHERE project_id = $1 AND to_visitor_id = ANY($2)", + ) + .bind(project_id).bind(others).bind(target).execute(&mut *tx).await?; + sqlx::query( + "INSERT INTO person_redirects (project_id, from_visitor_id, to_visitor_id) \ + SELECT $1, unnest($2::varchar[]), $3 \ + ON CONFLICT (project_id, from_visitor_id) DO UPDATE SET to_visitor_id = EXCLUDED.to_visitor_id", + ) + .bind(project_id).bind(others).bind(target).execute(&mut *tx).await?; + let redirects: Vec<(String,)> = sqlx::query_as( + "SELECT from_visitor_id FROM person_redirects WHERE project_id = $1 AND to_visitor_id = $2", + ) + .bind(project_id) + .bind(target) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let mut redis = state.redis.clone(); + for (from,) in redirects { + let _: Result<(), _> = redis + .hset(redirect_key(state, project_id), from, target) + .await; + } + Ok(json!({ "visitor_id": target, "merged": others, "rows_moved": moved })) +} + +/// Delete a person's profile; with `delete_events` also all of their data (GDPR). +pub async fn delete( + db: &PgPool, + project_id: Uuid, + visitor_id: &str, + delete_events: bool, +) -> AppResult { + require_person(db, project_id, visitor_id).await?; + if delete_events { + let r = privacy::delete_visitor_data(db, project_id, visitor_id).await?; + return Ok(serde_json::to_value(r).unwrap_or_else(|_| json!({ "ok": true }))); + } + let mut tx = db.begin().await?; + for sql in [ + "DELETE FROM user_aliases WHERE project_id = $1 AND visitor_id = $2", + "DELETE FROM user_profiles WHERE project_id = $1 AND visitor_id = $2", + "DELETE FROM cohort_people cp USING person_cohorts c WHERE cp.cohort_id = c.id AND c.project_id = $1 AND cp.visitor_id = $2", + ] { + sqlx::query(sql).bind(project_id).bind(visitor_id).execute(&mut *tx).await?; + } + tx.commit().await?; + Ok(json!({ "ok": true, "deleted_events": false })) +} diff --git a/crates/pulse-server/src/services/privacy.rs b/crates/pulse-server/src/services/privacy.rs index ab2174f..213302a 100644 --- a/crates/pulse-server/src/services/privacy.rs +++ b/crates/pulse-server/src/services/privacy.rs @@ -272,6 +272,12 @@ pub async fn export_visitor_data( "js_errors": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM js_errors WHERE project_id = $1 AND visitor_id = $2 ORDER BY created_at DESC) t", project_id, visitor_id).await?, "log_entries": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM log_entries WHERE project_id = $1 AND visitor_id = $2 ORDER BY created_at DESC) t", project_id, visitor_id).await?, "click_events": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM click_events WHERE project_id = $1 AND visitor_id = $2 ORDER BY created_at DESC) t", project_id, visitor_id).await?, + "feature_flag_evaluations": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM feature_flag_evaluations WHERE project_id = $1 AND visitor_id = $2) t", project_id, visitor_id).await?, + "guide_events": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM guide_events WHERE project_id = $1 AND visitor_id = $2) t", project_id, visitor_id).await?, + "event_quality_violations": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM event_quality_violations WHERE project_id = $1 AND visitor_id = $2) t", project_id, visitor_id).await?, + "llm_traces": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM llm_traces WHERE project_id = $1 AND visitor_id = $2) t", project_id, visitor_id).await?, + "account_memberships": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(to_jsonb(t)), '[]'::jsonb) FROM (SELECT * FROM account_memberships WHERE project_id = $1 AND visitor_id = $2) t", project_id, visitor_id).await?, + "cohort_memberships": fetch_json_array(db, "SELECT COALESCE(jsonb_agg(jsonb_build_object('cohort_id', c.id, 'cohort', c.name, 'added_at', cp.added_at)), '[]'::jsonb) FROM cohort_people cp JOIN person_cohorts c ON c.id = cp.cohort_id WHERE c.project_id = $1 AND cp.visitor_id = $2", project_id, visitor_id).await?, }); Ok(serde_json::json!({ @@ -294,30 +300,39 @@ pub async fn export_visitor_data( })) } +/// Every table keyed by `(project_id, visitor_id)`, children first. Used for +/// GDPR delete and person merge so no table is missed. +pub const VISITOR_TABLES: &[&str] = &[ + "session_recordings", + "survey_responses", + "experiment_assignments", + "experiment_exposures", + "goal_conversions", + "web_vitals", + "scroll_depths", + "search_queries", + "outlinks", + "js_errors", + "log_entries", + "click_events", + "feature_flag_evaluations", + "guide_events", + "event_quality_violations", + "llm_traces", + "account_memberships", + "pageviews", + "events", + "sessions", + "user_aliases", + "user_profiles", +]; + pub async fn delete_visitor_data( db: &PgPool, project_id: Uuid, visitor_id: &str, ) -> AppResult { - let tables = [ - "session_recordings", - "survey_responses", - "experiment_assignments", - "experiment_exposures", - "goal_conversions", - "web_vitals", - "scroll_depths", - "search_queries", - "outlinks", - "js_errors", - "log_entries", - "click_events", - "pageviews", - "events", - "sessions", - "user_aliases", - "user_profiles", - ]; + let tables = VISITOR_TABLES; let mut deleted = serde_json::Map::new(); for table in tables { @@ -329,6 +344,19 @@ pub async fn delete_visitor_data( .await?; deleted.insert(table.to_string(), serde_json::json!(result.rows_affected())); } + // Cohort memberships carry no project_id; scope them through the cohort. + let result = sqlx::query( + "DELETE FROM cohort_people cp USING person_cohorts c \ + WHERE cp.cohort_id = c.id AND c.project_id = $1 AND cp.visitor_id = $2", + ) + .bind(project_id) + .bind(visitor_id) + .execute(db) + .await?; + deleted.insert( + "cohort_people".to_string(), + serde_json::json!(result.rows_affected()), + ); Ok(DsarDeleteResult { visitor_id: visitor_id.to_string(), diff --git a/migrations/034_persons.down.sql b/migrations/034_persons.down.sql new file mode 100644 index 0000000..62b2225 --- /dev/null +++ b/migrations/034_persons.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS person_redirects; +DROP INDEX IF EXISTS idx_events_visitor; +DROP INDEX IF EXISTS idx_pageviews_visitor; diff --git a/migrations/034_persons.up.sql b/migrations/034_persons.up.sql new file mode 100644 index 0000000..0710987 --- /dev/null +++ b/migrations/034_persons.up.sql @@ -0,0 +1,13 @@ +-- Per-person lookups (person event feed, merge, GDPR delete) scan by visitor. +CREATE INDEX IF NOT EXISTS idx_pageviews_visitor ON pageviews(project_id, visitor_id, created_at); +CREATE INDEX IF NOT EXISTS idx_events_visitor ON events(project_id, visitor_id, created_at); + +-- Person merges: events still arriving under a merged-away visitor id are +-- attributed to the person they were merged into. +CREATE TABLE IF NOT EXISTS person_redirects ( + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + from_visitor_id VARCHAR(64) NOT NULL, + to_visitor_id VARCHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (project_id, from_visitor_id) +);