fix(analytics): distinguish fallback reasons + forward backend error message (SDK-79, SDK-83)

packages/mixpanel/lib/flags/index.js

@@ -5,8 +5,18 @@
 
 const LocalFeatureFlagsProvider = require("./local_flags");
 const RemoteFeatureFlagsProvider = require("./remote_flags");
+const {
+  VariantSource,
+  FallbackReason,
+  withSource,
+  asFallback,
+} = require("./variant_source");
 
 module.exports = {
   LocalFeatureFlagsProvider,
   RemoteFeatureFlagsProvider,
+  VariantSource,
+  FallbackReason,
+  withSource,
+  asFallback,
 };

packages/mixpanel/lib/flags/local_flags.js

@@ -17,6 +17,12 @@ const {
   lowercaseAllKeysAndValues,
   lowercaseLeafNodes,
 } = require("./utils");
+const {
+  VariantSource,
+  FallbackReason,
+  withSource,
+  asFallback,
+} = require("./variant_source");
 const { apply } = require("json-logic-js");
 
 class LocalFeatureFlagsProvider extends FeatureFlagsProvider {
@@ -151,15 +157,18 @@ class LocalFeatureFlagsProvider extends FeatureFlagsProvider {
     const flag = this.flagDefinitions.get(flagKey);
 
     if (!flag) {
-      this.logger?.warn(`Cannot find flag definition for key: '${flagKey}`);
-      return fallbackVariant;
+      this.logger?.warn(`Cannot find flag definition for key: '${flagKey}'`);
+      return asFallback(fallbackVariant, FallbackReason.flagNotFound());
     }
 
     if (!Object.hasOwn(context, flag.context)) {
       this.logger?.warn(
         `The variant assignment key, '${flag.context}' for flag, '${flagKey}' is not present in the supplied user context dictionary`,
       );
-      return fallbackVariant;
+      return asFallback(
+        fallbackVariant,
+        FallbackReason.missingContextKey(flag.context),
+      );
     }
 
     const contextValue = context[flag.context];
@@ -185,10 +194,10 @@ class LocalFeatureFlagsProvider extends FeatureFlagsProvider {
       if (reportExposure) {
         this.trackExposureEvent(flagKey, selectedVariant, context);
       }
-      return selectedVariant;
+      return withSource(selectedVariant, VariantSource.LOCAL);
     }
 
-    return fallbackVariant;
+    return asFallback(fallbackVariant, FallbackReason.noRolloutMatch());
   }
 
   /**
@@ -199,10 +208,11 @@ class LocalFeatureFlagsProvider extends FeatureFlagsProvider {
    */
   getAllVariants(context) {
     const variants = {};
+    const fallback = { variant_value: null };
 
     for (const flagKey of this.flagDefinitions.keys()) {
-      const variant = this.getVariant(flagKey, null, context, false);
-      if (variant !== null) {
+      const variant = this.getVariant(flagKey, fallback, context, false);
+      if (variant.variant_source === VariantSource.LOCAL) {
         variants[flagKey] = variant;
       }
     }

packages/mixpanel/lib/flags/remote_flags.js

@@ -11,6 +11,12 @@
  */
 
 const FeatureFlagsProvider = require("./flags");
+const {
+  VariantSource,
+  FallbackReason,
+  withSource,
+  asFallback,
+} = require("./variant_source");
 
 class RemoteFeatureFlagsProvider extends FeatureFlagsProvider {
   /**
@@ -86,19 +92,36 @@ class RemoteFeatureFlagsProvider extends FeatureFlagsProvider {
       const flags = response.flags || {};
       const selectedVariant = flags[flagKey];
       if (!selectedVariant) {
-        return fallbackVariant;
+        // The /flags endpoint only returns variants the user is enrolled in,
+        // so a missing key could mean the flag doesn't exist OR the user
+        // isn't in any rollout. The remote SDK can't tell them apart without
+        // server-side help — surface as FLAG_NOT_FOUND for now.
+        return asFallback(fallbackVariant, FallbackReason.flagNotFound());
       }
 
       if (reportExposure) {
         this.trackExposureEvent(flagKey, selectedVariant, context, latencyMs);
       }
 
-      return selectedVariant;
+      return withSource(selectedVariant, VariantSource.REMOTE);
     } catch (err) {
+      // catch(err) can bind any thrown value — including null/undefined/plain
+      // strings — so `err.message` isn't safe to read directly. Extract once,
+      // reuse for both logging and the FallbackReason payload.
+      const errorMessage =
+        err instanceof Error ? err.message : String(err ?? "unknown error");
       this.logger?.error(
-        `Failed to get variant for flag '${flagKey}': ${err.message}`,
+        `Failed to get variant for flag '${flagKey}': ${errorMessage}`,
+      );
+      // SDK-83: attach the error message so the OpenFeature wrapper can
+      // forward it as errorMessage instead of swallowing the cause into
+      // a bare GENERAL error. The backend's response body (e.g.
+      // "distinct_id must be provided in evalContext as a string") arrives
+      // on err.message via the HTTP layer.
+      return asFallback(
+        fallbackVariant,
+        FallbackReason.backendError(errorMessage),
       );
-      return fallbackVariant;
     }
   }
 
@@ -131,7 +154,12 @@ class RemoteFeatureFlagsProvider extends FeatureFlagsProvider {
   async getAllVariants(context) {
     try {
       const response = await this._fetchFlags(context);
-      return response.flags || {};
+      const flags = response.flags || {};
+      const tagged = {};
+      for (const [key, variant] of Object.entries(flags)) {
+        tagged[key] = withSource(variant, VariantSource.REMOTE);
+      }
+      return tagged;
     } catch (err) {
       this.logger?.error(`Failed to get all remote variants: ${err.message}`);
       return null;

packages/mixpanel/lib/flags/types.d.ts

@@ -109,12 +109,62 @@ export interface ExperimentationFlag {
   hash_salt?: string;
 }
 
+/**
+ * Where a {@link SelectedVariant} came from. Set by the providers on every
+ * returned variant. Coarse-grained — see {@link FallbackReason} for the
+ * specific reason behind a fallback.
+ */
+export type VariantSource = "local" | "remote" | "fallback";
+
+export const VariantSource: {
+  LOCAL: "local";
+  REMOTE: "remote";
+  FALLBACK: "fallback";
+};
+
+/**
+ * Why the SDK returned the developer fallback. Only meaningful when
+ * `variant_source === 'fallback'`.
+ *
+ * `kind` is the discriminator (PHP-aligned). `message` is set on reasons
+ * that carry useful detail (BACKEND_ERROR with the backend's response body,
+ * MISSING_CONTEXT_KEY with the missing attribute name); null otherwise.
+ * The OpenFeature wrapper dispatches on kind and forwards message into
+ * ResolutionDetails.errorMessage.
+ */
+export type FallbackReasonKind =
+  | "FLAG_NOT_FOUND"
+  | "MISSING_CONTEXT_KEY"
+  | "NO_ROLLOUT_MATCH"
+  | "BACKEND_ERROR";
+
+export interface FallbackReason {
+  kind: FallbackReasonKind;
+  message: string | null;
+}
+
+export const FallbackReason: {
+  Kind: {
+    FLAG_NOT_FOUND: "FLAG_NOT_FOUND";
+    MISSING_CONTEXT_KEY: "MISSING_CONTEXT_KEY";
+    NO_ROLLOUT_MATCH: "NO_ROLLOUT_MATCH";
+    BACKEND_ERROR: "BACKEND_ERROR";
+  };
+  flagNotFound(): FallbackReason;
+  noRolloutMatch(): FallbackReason;
+  missingContextKey(key?: string | null): FallbackReason;
+  backendError(message: string): FallbackReason;
+};
+
 export interface SelectedVariant {
   variant_key?: string | null;
   variant_value: any;
   experiment_id?: string;
   is_experiment_active?: boolean;
   is_qa_tester?: boolean;
+  variant_source?: VariantSource;
+  /** undefined on success; set when variant_source is 'fallback'. */
+  fallback_reason?: FallbackReason;
 }
 
 export interface RemoteFlagsResponse {

packages/mixpanel/lib/flags/variant_source.js

@@ -0,0 +1,80 @@
+/**
+ * Where a SelectedVariant came from. Set by the providers on every returned
+ * variant. Coarse-grained (local / remote / fallback) — for the specific
+ * reason behind a fallback, see FallbackReason.
+ */
+const VariantSource = Object.freeze({
+  LOCAL: "local",
+  REMOTE: "remote",
+  FALLBACK: "fallback",
+});
+
+/**
+ * Why the SDK returned the developer fallback. Only meaningful when
+ * `variant_source === 'fallback'`.
+ *
+ * `kind` is the discriminator (PHP-aligned). `message` is set on the reasons
+ * that carry useful detail (BACKEND_ERROR with the backend's response body,
+ * MISSING_CONTEXT_KEY with the missing attribute name); null otherwise. The
+ * OpenFeature wrapper dispatches on kind and forwards message into
+ * ResolutionDetails.errorMessage.
+ */
+const FallbackReasonKind = Object.freeze({
+  FLAG_NOT_FOUND: "FLAG_NOT_FOUND",
+  MISSING_CONTEXT_KEY: "MISSING_CONTEXT_KEY",
+  NO_ROLLOUT_MATCH: "NO_ROLLOUT_MATCH",
+  BACKEND_ERROR: "BACKEND_ERROR",
+});
+
+const FallbackReason = Object.freeze({
+  flagNotFound() {
+    return _FLAG_NOT_FOUND;
+  },
+  noRolloutMatch() {
+    return _NO_ROLLOUT_MATCH;
+  },
+  missingContextKey(key = null) {
+    return Object.freeze({
+      kind: FallbackReasonKind.MISSING_CONTEXT_KEY,
+      message: key,
+    });
+  },
+  backendError(message) {
+    return Object.freeze({ kind: FallbackReasonKind.BACKEND_ERROR, message });
+  },
+  // Re-exported so consumers can compare via `reason.kind === FallbackReason.Kind.BACKEND_ERROR`
+  Kind: FallbackReasonKind,
+});
+
+const _FLAG_NOT_FOUND = Object.freeze({
+  kind: FallbackReasonKind.FLAG_NOT_FOUND,
+  message: null,
+});
+const _NO_ROLLOUT_MATCH = Object.freeze({
+  kind: FallbackReasonKind.NO_ROLLOUT_MATCH,
+  message: null,
+});
+
+/**
+ * Return a shallow copy of `variant` tagged with `source`. Clears
+ * fallback_reason — use asFallback when returning a fallback.
+ */
+function withSource(variant, source) {
+  return Object.assign({}, variant, {
+    variant_source: source,
+    fallback_reason: undefined,
+  });
+}
+
+/**
+ * Return a shallow copy of `variant` tagged as a fallback with `reason`.
+ * `reason` is a FallbackReason value object from one of the factories above.
+ */
+function asFallback(variant, reason) {
+  return Object.assign({}, variant, {
+    variant_source: VariantSource.FALLBACK,
+    fallback_reason: reason,
+  });
+}
+
+module.exports = { VariantSource, FallbackReason, withSource, asFallback };

packages/mixpanel/test/flags/local_flags.js

@@ -1,5 +1,9 @@
 const nock = require("nock");
 const LocalFeatureFlagsProvider = require("../../lib/flags/local_flags");
+const {
+  VariantSource,
+  FallbackReason,
+} = require("../../lib/flags/variant_source");
 
 const mockFlagDefinitionsResponse = (flags) => {
   const response = {
@@ -664,6 +668,52 @@ describe("LocalFeatureFlagsProvider", () => {
       provider.getVariant(FLAG_KEY, FALLBACK, { company_id: "company123" });
       expect(mockTracker).not.toHaveBeenCalled();
     });
+
+    // SDK-79: every fallback path must be tagged distinctly via
+    // variant_source + fallback_reason so the OpenFeature wrapper can map
+    // each to the spec-correct error code.
+    describe("variant_source / fallback_reason tagging", () => {
+      it("tags matched variants as local with no fallback_reason", async () => {
+        await createFlagAndLoadItIntoSDK(
+          { rolloutPercentage: 100.0 },
+          provider,
+        );
+        const result = provider.getVariant(FLAG_KEY, FALLBACK, TEST_CONTEXT);
+        expect(result.variant_source).toBe(VariantSource.LOCAL);
+        expect(result.fallback_reason).toBeUndefined();
+        expect(result.variant_key).toBeDefined();
+      });
+
+      it("tags missing flag as fallback / FLAG_NOT_FOUND", async () => {
+        mockFlagDefinitionsResponse([]);
+        await provider.startPollingForDefinitions();
+        const result = provider.getVariant("missing", FALLBACK, TEST_CONTEXT);
+        expect(result.variant_source).toBe(VariantSource.FALLBACK);
+        expect(result.fallback_reason.kind).toBe(
+          FallbackReason.Kind.FLAG_NOT_FOUND,
+        );
+        expect(result.fallback_reason.message).toBeNull();
+      });
+
+      it("tags missing context as fallback / MISSING_CONTEXT_KEY with the missing key", async () => {
+        await createFlagAndLoadItIntoSDK({ context: "distinct_id" }, provider);
+        const result = provider.getVariant(FLAG_KEY, FALLBACK, {});
+        expect(result.variant_source).toBe(VariantSource.FALLBACK);
+        expect(result.fallback_reason.kind).toBe(
+          FallbackReason.Kind.MISSING_CONTEXT_KEY,
+        );
+        expect(result.fallback_reason.message).toBe("distinct_id");
+      });
+
+      it("tags no-rollout-match as fallback / NO_ROLLOUT_MATCH", async () => {
+        await createFlagAndLoadItIntoSDK({ rolloutPercentage: 0.0 }, provider);
+        const result = provider.getVariant(FLAG_KEY, FALLBACK, TEST_CONTEXT);
+        expect(result.variant_source).toBe(VariantSource.FALLBACK);
+        expect(result.fallback_reason.kind).toBe(
+          FallbackReason.Kind.NO_ROLLOUT_MATCH,
+        );
+      });
+    });
   });
 
   describe("getAllVariants", () => {