{
  "page": {
    "id": "xg1sy4d9c1r0",
    "name": "Air",
    "url": "https://status.air.inc"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","group_id":null,"page_id":"xg1sy4d9c1r0"},{"status":"operational","name":"API","created_at":"2023-12-14T19:44:08.807Z","updated_at":"2026-05-02T16:32:07.071Z","position":2,"description":"API for the Air web application","showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"8244vgbk9hjw","group_id":null,"page_id":"xg1sy4d9c1r0"},{"status":"operational","name":"Air Flow Desktop Sync App","created_at":"2025-10-01T13:55:08.506Z","updated_at":"2026-05-02T16:32:07.080Z","position":3,"description":null,"showcase":true,"start_date":"2025-08-01T00:00:00.000Z","id":"fmbrkz71lqb6","group_id":null,"page_id":"xg1sy4d9c1r0"}],
  "incidents": [{"name":"Media processing degredation","status":"resolved","created_at":"2026-05-02T15:50:47.728Z","updated_at":"2026-05-02T16:32:07.114Z","monitoring_at":null,"resolved_at":"2026-05-02T16:32:07.102Z","impact":"minor","shortlink":"https://stspg.io/l9b4lxzyf5v8","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-05-02T15:50:47.720Z","reminder_intervals":null,"id":"5ftv8493vbxw","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"This incident has been resolved. All systems are operational.","created_at":"2026-05-02T16:32:07.102Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-02T16:32:07.102Z","display_at":"2026-05-02T16:32:07.102Z","deliver_notifications":false,"tweet_id":null,"id":"ds1yt96n0l4f","incident_id":"5ftv8493vbxw","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"degraded_performance","new_status":"operational"},{"code":"8244vgbk9hjw","name":"API","old_status":"degraded_performance","new_status":"operational"},{"code":"fmbrkz71lqb6","name":"Air Flow Desktop Sync App","old_status":"degraded_performance","new_status":"operational"}]},{"status":"investigating","body":"We are investigating an issue that is causing degraded performance on media processing. We will update the status of this investigation here as we know more.","created_at":"2026-05-02T15:50:47.878Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-02T15:50:47.878Z","display_at":"2026-05-02T15:50:47.878Z","deliver_notifications":false,"tweet_id":null,"id":"kbzmk2l0xlkn","incident_id":"5ftv8493vbxw","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"degraded_performance"},{"code":"8244vgbk9hjw","name":"API","old_status":"operational","new_status":"degraded_performance"},{"code":"fmbrkz71lqb6","name":"Air Flow Desktop Sync App","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null},{"status":"operational","name":"API","created_at":"2023-12-14T19:44:08.807Z","updated_at":"2026-05-02T16:32:07.071Z","position":2,"description":"API for the Air web application","showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"8244vgbk9hjw","page_id":"xg1sy4d9c1r0","group_id":null},{"status":"operational","name":"Air Flow Desktop Sync App","created_at":"2025-10-01T13:55:08.506Z","updated_at":"2026-05-02T16:32:07.080Z","position":3,"description":null,"showcase":true,"start_date":"2025-08-01T00:00:00.000Z","id":"fmbrkz71lqb6","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"App Unresponsive","status":"resolved","created_at":"2025-11-03T14:33:50.728Z","updated_at":"2025-11-03T14:43:37.153Z","monitoring_at":null,"resolved_at":"2025-11-03T14:43:37.138Z","impact":"major","shortlink":"https://stspg.io/9nwlr8nv878z","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-11-03T14:33:50.722Z","reminder_intervals":null,"id":"483k2b8rp49z","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"This incident has now been resolved. The app should be fully responsive and operating normally again.","created_at":"2025-11-03T14:43:37.138Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-03T14:43:37.138Z","display_at":"2025-11-03T14:43:37.138Z","deliver_notifications":true,"tweet_id":null,"id":"bwmj22vdlp2p","incident_id":"483k2b8rp49z","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"degraded_performance","new_status":"operational"}]},{"status":"identified","body":"We’ve identified the underlying cause of the issue and are implementing a fix. Some users may continue to experience slow load times while we work to restore service.","created_at":"2025-11-03T14:41:30.796Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-03T14:41:30.796Z","display_at":"2025-11-03T14:41:30.796Z","deliver_notifications":true,"tweet_id":null,"id":"brpxd4rdw2kr","incident_id":"483k2b8rp49z","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"partial_outage","new_status":"degraded_performance"}]},{"status":"investigating","body":"We’re currently aware of an issue causing the app to be unresponsive for some users. Our engineering team is investigating and working to restore full functionality as quickly as possible.","created_at":"2025-11-03T14:33:50.815Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-03T14:33:50.815Z","display_at":"2025-11-03T14:33:50.815Z","deliver_notifications":true,"tweet_id":null,"id":"0pkn9ng4nwr4","incident_id":"483k2b8rp49z","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"AWS Outage Impacting Service","status":"resolved","created_at":"2025-10-20T12:57:17.432Z","updated_at":"2025-10-21T13:25:16.695Z","monitoring_at":"2025-10-20T13:03:04.113Z","resolved_at":"2025-10-21T13:25:16.678Z","impact":"critical","shortlink":"https://stspg.io/dtwmbns0lc3p","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"critical","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-10-20T12:57:17.427Z","reminder_intervals":null,"id":"srv338x99mrp","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"The underlying AWS issues have been resolved, and our systems are mostly back to normal. Uploads and processing are functioning as expected, but we’re still working through a backlog of assets created during the outage. As we catch up, you may notice some delays in enrichment or processing times, but these should continue to improve throughout the day.","created_at":"2025-10-21T13:25:16.678Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-21T13:25:16.678Z","display_at":"2025-10-21T13:25:16.678Z","deliver_notifications":true,"tweet_id":null,"id":"b55ysxs07898","incident_id":"srv338x99mrp","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"partial_outage","new_status":"operational"}]},{"status":"monitoring","body":"We’re still seeing some issues with processing uploaded content, but there’s been slight improvement as AWS continues to recover from their US-EAST-1 outage. Processing may still be slower than usual. We’re staying in close contact with AWS Support as they work to fully restore capacity.","created_at":"2025-10-20T19:04:32.589Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-20T19:04:32.589Z","display_at":"2025-10-20T19:04:32.589Z","deliver_notifications":true,"tweet_id":null,"id":"cwtdxbxkzqvg","incident_id":"srv338x99mrp","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"major_outage","new_status":"partial_outage"}]},{"status":"monitoring","body":"AWS is still experiencing issues in its US-EAST-1 region, which is causing delays in processing uploaded content. We’re in touch with AWS Support and working to fix the problem and restore normal performance.","created_at":"2025-10-20T14:33:51.053Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-20T14:33:51.053Z","display_at":"2025-10-20T14:33:51.053Z","deliver_notifications":true,"tweet_id":null,"id":"kty63hgdkfy8","incident_id":"srv338x99mrp","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"major_outage","new_status":"major_outage"}]},{"status":"monitoring","body":"We’re seeing delayed processing of uploaded content. Uploads are completing successfully, but due to the ongoing AWS US-EAST-1 outage, our processing capacity is being throttled by AWS. Our engineering team is actively working with AWS Support, who have confirmed they’re working to restore full capacity.","created_at":"2025-10-20T13:03:04.113Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-20T13:03:04.113Z","display_at":"2025-10-20T13:03:04.113Z","deliver_notifications":true,"tweet_id":null,"id":"dhtdqdw24254","incident_id":"srv338x99mrp","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"major_outage","new_status":"major_outage"}]},{"status":"identified","body":"We are continuing to work on a fix for this issue.","created_at":"2025-10-20T12:58:34.714Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-20T12:58:34.714Z","display_at":"2025-10-20T12:58:34.714Z","deliver_notifications":true,"tweet_id":null,"id":"cb8x27xqjlmb","incident_id":"srv338x99mrp","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"major_outage"}]},{"status":"identified","body":"We are currently experiencing an outage impacting parts of our infrastructure due to an ongoing AWS US-EAST-1 regional issue. This may cause degraded performance, delayed processing, or limited access for some users. Our engineering team is actively monitoring AWS’s recovery progress and working to mitigate impact.","created_at":"2025-10-20T12:57:17.472Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-20T12:57:17.472Z","display_at":"2025-10-20T12:57:17.472Z","deliver_notifications":true,"tweet_id":null,"id":"l9jhq1kjf0zh","incident_id":"srv338x99mrp","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"Elevated Response Times","status":"resolved","created_at":"2025-10-07T17:52:52.147Z","updated_at":"2025-10-07T18:05:46.408Z","monitoring_at":null,"resolved_at":"2025-10-07T18:05:46.391Z","impact":"minor","shortlink":"https://stspg.io/6t8j996ws893","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-10-07T17:52:52.142Z","reminder_intervals":null,"id":"vbr3608c2z19","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"The response time slowness has been resolved.","created_at":"2025-10-07T18:05:46.391Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-07T18:05:46.391Z","display_at":"2025-10-07T18:05:46.391Z","deliver_notifications":true,"tweet_id":null,"id":"kp6mnv0prf6y","incident_id":"vbr3608c2z19","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"degraded_performance","new_status":"operational"}]},{"status":"investigating","body":"System response times are currently degraded, causing slower performance for users. The team is actively investigating the cause and working to restore normal service levels.Retry","created_at":"2025-10-07T17:52:52.218Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-07T17:52:52.218Z","display_at":"2025-10-07T17:52:52.218Z","deliver_notifications":true,"tweet_id":null,"id":"2rfdng888d79","incident_id":"vbr3608c2z19","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"Application Loading Issues","status":"resolved","created_at":"2025-10-02T17:42:10.000Z","updated_at":"2025-10-09T22:12:57.771Z","monitoring_at":null,"resolved_at":"2025-10-02T17:59:03.000Z","impact":"minor","shortlink":"https://stspg.io/5vprqnv1p1s3","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-10-02T17:42:10.000Z","reminder_intervals":null,"id":"h46qbj5ct6g4","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2025-10-02T21:19:03.140Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-02T21:21:11.485Z","display_at":"2025-10-02T17:59:03.000Z","deliver_notifications":true,"tweet_id":null,"id":"36p781zb1k7k","incident_id":"h46qbj5ct6g4","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"degraded_performance","new_status":"operational"}]},{"status":"investigating","body":"We are currently investigating reports that users are unable to load the application. Our team is actively looking into the issue and we will provide an update as soon as we have more information.","created_at":"2025-10-02T18:03:10.340Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-02T21:20:56.150Z","display_at":"2025-10-02T17:42:10.000Z","deliver_notifications":true,"tweet_id":null,"id":"f626kdx96ycp","incident_id":"h46qbj5ct6g4","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":"### Overview\n\nIncident name: Ineffective planning of isolated db queries led to degraded app experience  \nDate and time: 2025-10-02 13:47–14:01 ET  \nAffected areas: Air’s web app  \nStatus: Resolved\n\n### Customer impact\n\n* Some users experienced slow loading or were unable to load the Air app for roughly 15 minutes. The issue was fully resolved the same day.\n* Time window: Approximately 1:47–2:01 PM ET on October 2, 2025\n* Data and security: No data loss or security exposure occurred\n\n### What happened\n\nMultiple long-running queries caused contention on the primary database, leading to app unavailability and elevated error rates until mitigation reduced load and queries resumed normal performance.\n\n### Root cause\n\n* Primary cause: On the writer, the query planner failed to use an index on the clip table for a frequently executed query, resulting in full table scans and spills to disk under load.\n\n### Timeline \\(high level\\)\n\n* 13:47: Degradation reported; app fails to load.\n* 13:48: Error rate confirmed elevated; incident channel started.\n* 14:00: Error rates decrease and app loads.\n* 14:01: Huddle begins; root cause investigation continues.\n* 2025-10-03 04:56: Database parameter change applied to reduce spill risk for heavy queries.\n\n### Preventative actions\n\n* Immediate fixes completed\n\n    * Reduced load and stabilized the app within minutes\n    * Completed database maintenance and configuration updates to improve query planning and performance\n    \n* Near-term improvements \\(planned or in progress\\)\n\n    * Enhanced autoscaling of database readers.\n    * Configure and tune autovacuum to reduce bloat.\n    * Timeout configuration for Lambda.\n    \n\n### Frequently asked questions\n\n* Was any customer data lost?\n\n    * No. We confirmed no data loss or security exposure.\n    \n* Do customers need to take any action?\n\n    * No. Systems recovered without degradation. If anything looks off, contact us and we will investigate immediately.\n    \n* Could this happen again?\n\n    * The underlying query inefficiency has been mitigated and additional monitoring and runbook improvements are in progress.\n    \n\n### Need help?\n\nIf you notice anything unexpected, please reach out to your Air contact or reply to your most recent support thread and we’ll follow up immediately.","postmortem_body_last_updated_at":"2025-10-09T22:12:57.756Z","postmortem_ignored":false,"postmortem_published_at":"2025-10-09T22:12:57.756Z","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"App is getting 504 Gateway timeouts","status":"resolved","created_at":"2025-09-30T16:45:06.000Z","updated_at":"2025-10-06T19:37:33.172Z","monitoring_at":null,"resolved_at":"2025-09-30T17:45:40.000Z","impact":"major","shortlink":"https://stspg.io/jn0j372z0cn4","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-09-30T16:45:06.000Z","reminder_intervals":null,"id":"pd1mkr4ff2m7","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2025-09-30T17:57:40.047Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-30T18:00:01.958Z","display_at":"2025-09-30T17:45:40.000Z","deliver_notifications":true,"tweet_id":null,"id":"6467ryfjl9wl","incident_id":"pd1mkr4ff2m7","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"degraded_performance","new_status":"operational"}]},{"status":"identified","body":"The issue has been identified and a fix is being implemented.","created_at":"2025-09-30T17:46:45.389Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-30T18:00:00.232Z","display_at":"2025-09-30T17:25:45.000Z","deliver_notifications":true,"tweet_id":null,"id":"hbnh1xftr4pt","incident_id":"pd1mkr4ff2m7","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"major_outage","new_status":"degraded_performance"}]},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2025-09-30T16:58:06.643Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-30T16:58:45.227Z","display_at":"2025-09-30T16:45:06.000Z","deliver_notifications":true,"tweet_id":null,"id":"713sfgwnslhl","incident_id":"pd1mkr4ff2m7","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"major_outage"}]}],"postmortem_body":"### **Overview**\n\n* Incident name: Internal Cleanup Led to API Request Handling Issues\n* Date and time: 2025-09-30, roughly 12:45 PM–1:45 PM ET\n* Affected areas: Consumers of Air’s API\n* Status: Resolved\n\n### **Customer impact**\n\n* What customers experienced: During this period foreground apps were left with intermittent success in accessing the api with more impact seen on operations modifying data\n* Scope: Actions that modified data and intermittent impact to actions reading data. Some background tasks such as media processing, AI enrichments, indexing, and downloads were delayed during the incident but resumed following resolution without degradation.\n* Duration: 2025-09-30, roughly 12:45 PM–1:45 PM ET\n* Data and security: No data loss or security exposure occurred.\n\n### **What happened**\n\nInternal cleanup on a dataset led to increased load on Air’s primary database which led to subsequent queries being impacted \\(latency and/or timing out\\).\n\n### **Root cause**\n\n* Primary cause: cleanup logic was not throttled correctly to limit load it placed on the primary database.\n\n### **Timeline \\(high level\\)**\n\n* 12:45 PM: Degradation from cleanup observed and cleanup terminated\n* 1:00 PM: Load on primary decreases and intermittent api access is observed\n* 1:08 PM: longer running queries that were created from cleanup terminated\n* 1:25 PM: Additional read capacity added to the database to support volume of requests during recovery\n* 1:45 PM: All app functionality returned to nominal behavior and async processing resumed\n\n### **Preventative actions**\n\n* Immediate fixes completed\n\n    * Increased environment capacity to handle load\n    * Cleanup process placed back in review for further refinement\n    \n* Near-term improvements\n\n    * Refine approval and application out-of-band processing \\(e.g cleanup, etc\\)\n    * Alert on early indicators \\(e.g., database locks, etc\\) for faster detection\n    \n* Long-term investments\n\n    * Dedicated system for out-of-band processing to provide additional guardrails\n    \n\n### **Frequently asked questions**\n\n* Was any customer data lost?\n\n    * No. We confirmed no data loss or security exposure.\n    \n* Do customers need to take any action?\n\n    * No. All queued work has been safely processed. If anything still looks off, please let us know and we will investigate immediately.\n    \n* How will we keep you updated?\n\n    * Your account team will share any follow-ups on improvements. We will also post future status updates through our standard channels if needed.","postmortem_body_last_updated_at":"2025-10-06T19:37:33.157Z","postmortem_ignored":false,"postmortem_published_at":"2025-10-06T19:37:33.157Z","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"Air is not responsive","status":"resolved","created_at":"2025-09-19T14:23:24.718Z","updated_at":"2025-09-19T17:07:19.573Z","monitoring_at":"2025-09-19T15:42:24.462Z","resolved_at":"2025-09-19T17:07:19.556Z","impact":"major","shortlink":"https://stspg.io/rtmsj6fm8151","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-09-19T14:23:24.712Z","reminder_intervals":null,"id":"mftmzyvlmp3b","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2025-09-19T17:07:19.556Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-19T17:07:19.556Z","display_at":"2025-09-19T17:07:19.556Z","deliver_notifications":true,"tweet_id":null,"id":"t4j170199ln8","incident_id":"mftmzyvlmp3b","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"operational"},{"code":"8244vgbk9hjw","name":"API","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"A fix has been implemented and we are monitoring the results. Air's web app is restored.","created_at":"2025-09-19T15:42:24.462Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-19T15:42:24.462Z","display_at":"2025-09-19T15:42:24.462Z","deliver_notifications":true,"tweet_id":null,"id":"82gk6xbfp6xq","incident_id":"mftmzyvlmp3b","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"partial_outage","new_status":"operational"},{"code":"8244vgbk9hjw","name":"API","old_status":"operational","new_status":"operational"}]},{"status":"identified","body":"API services have been restored. We are seeing delays with media processing, search reindexing, and notifications which is being actively investigated.","created_at":"2025-09-19T14:39:38.372Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-19T14:39:38.372Z","display_at":"2025-09-19T14:39:38.372Z","deliver_notifications":true,"tweet_id":null,"id":"sd93w2fcd3xd","incident_id":"mftmzyvlmp3b","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"major_outage","new_status":"partial_outage"},{"code":"8244vgbk9hjw","name":"API","old_status":"major_outage","new_status":"operational"}]},{"status":"identified","body":"Air's app is beginning to load for users. Uploading and notifications remains in a degraded state as Air continues implementing a fix.","created_at":"2025-09-19T14:36:36.365Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-19T14:36:36.365Z","display_at":"2025-09-19T14:36:36.365Z","deliver_notifications":true,"tweet_id":null,"id":"phpktmgkd74f","incident_id":"mftmzyvlmp3b","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"major_outage","new_status":"major_outage"},{"code":"8244vgbk9hjw","name":"API","old_status":"major_outage","new_status":"major_outage"}]},{"status":"identified","body":"The issue has been identified and a fix is being implemented.","created_at":"2025-09-19T14:33:25.240Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-19T14:33:25.240Z","display_at":"2025-09-19T14:33:25.240Z","deliver_notifications":true,"tweet_id":null,"id":"by60s63p5qh9","incident_id":"mftmzyvlmp3b","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"major_outage","new_status":"major_outage"},{"code":"8244vgbk9hjw","name":"API","old_status":"major_outage","new_status":"major_outage"}]},{"status":"investigating","body":"We are currently investigating an issue where Air's web interface is unresponsive.","created_at":"2025-09-19T14:23:24.806Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-19T14:23:24.806Z","display_at":"2025-09-19T14:23:24.806Z","deliver_notifications":true,"tweet_id":null,"id":"nngkvx3z56hn","incident_id":"mftmzyvlmp3b","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"major_outage"},{"code":"8244vgbk9hjw","name":"API","old_status":"operational","new_status":"major_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null},{"status":"operational","name":"API","created_at":"2023-12-14T19:44:08.807Z","updated_at":"2026-05-02T16:32:07.071Z","position":2,"description":"API for the Air web application","showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"8244vgbk9hjw","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"Reindexing/Uploading Issues","status":"resolved","created_at":"2025-09-10T18:43:19.236Z","updated_at":"2025-09-16T21:08:40.231Z","monitoring_at":"2025-09-11T01:02:35.547Z","resolved_at":"2025-09-11T01:28:44.527Z","impact":"minor","shortlink":"https://stspg.io/vwmzymk5gt07","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-09-10T18:43:19.231Z","reminder_intervals":null,"id":"htq6kk5gh942","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2025-09-11T01:28:44.527Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-11T01:28:44.527Z","display_at":"2025-09-11T01:28:44.527Z","deliver_notifications":true,"tweet_id":null,"id":"qtmp52fjmn9n","incident_id":"htq6kk5gh942","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"We are starting to see tasks being processed again. Search indexing is slightly delayed, which is expected given the current processing load. While recovery has begun, we still have a backlog of tasks to reprocess and are continuing to monitor until we are fully in the clear.","created_at":"2025-09-11T01:02:35.547Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-11T01:02:35.547Z","display_at":"2025-09-11T01:02:35.547Z","deliver_notifications":true,"tweet_id":null,"id":"rgjc86wmfpps","incident_id":"htq6kk5gh942","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"degraded_performance","new_status":"operational"}]},{"status":"investigating","body":"We’re currently investigating an issue where reindexing and uploading are not working in some workspaces. Our engineering team is working to identify the cause and restore full functionality as quickly as possible.","created_at":"2025-09-10T18:43:19.313Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-10T18:43:19.313Z","display_at":"2025-09-10T18:43:19.313Z","deliver_notifications":true,"tweet_id":null,"id":"w0k82h29rbzw","incident_id":"htq6kk5gh942","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":"### Overview\n\n* Incident name: Delayed media asset processing\n* Date and time: 2025-09-10, roughly 2:03 PM–6:10 PM ET \\(with safe reprocessing continuing through 12:30 AM on 9/11\\)\n* Affected areas: Background processing of certain actions such as search reindexing, upload processing, and zip downloads. It’s worth clarifying that uploading originals were functioning as usual.\n* Status: Resolved\n\n### Customer impact\n\n* What customers experienced: Some background tasks such as media processing, AI enrichments, indexing, and downloads were delayed. Foreground app access remained available.\n* Scope: A subset of actions across customers experienced delays; not every action was impacted.\n* Duration: Delays primarily between ~2:03 PM and ~6:10 PM ET on 9/10. Backlogs were fully cleared by ~12:30 AM on 9/11.\n* Data and security: No data loss or security exposure occurred.\n\n### What happened\n\nA spike in background activity coincided with limited available infrastructure capacity and a rolling deployment. This combination created a large queue of background work. While service remained available, some background tasks took longer than normal to start and complete. We mitigated by increasing capacity, adjusting infrastructure to remove bottlenecks, and carefully reprocessing queued work to restore expected performance.\n\n### Root cause\n\n* Primary cause: Background work was scheduled across many dynamic queues. With capacity already at its maximum, a sudden increase in queued work reduced per-queue processing rate, allowing a backlog to build and slow processing further.\n* Contributing factors: Rolling deployment temporarily reduced available workers, and incoming event volume was significantly above typical peaks.\n\n### Timeline \\(high level\\)\n\n* 2:03 PM: Issue reported; we observed a growing background queue\n* 2:34 PM: Incident declared; investigation and mitigation began\n* 3:15 PM: First mitigation reduced the queue briefly; additional steps identified\n* 5:10 PM: Second mitigation increased capacity and stabilized the system\n* 6:10 PM: Backlog returned to normal levels; background processing restored\n* 12:30 AM \\(9/11\\): All queued background work safely reprocessed\n\n### Preventative actions\n\n* Immediate fixes completed\n\n    * Increased environment capacity and tuned scaling to better handle spikes\n    * Adjusted deployment approach to avoid temporary under-provisioning during rollouts\n    \n* Near-term improvements\n\n    * Add safeguards to prevent queue growth from reducing throughput\n    * Alert on early indicators \\(e.g., concurrent queue count and processing rate\\) for faster detection\n    \n* Long-term investments\n\n    * Optimize scheduling to keep processing performance stable as the number of queues grows\n    * Evaluate architecture options \\(partitioning and mediated queueing\\) for additional resilience\n    \n\n### Frequently asked questions\n\n* Was any customer data lost?\n\n    * No. We confirmed no data loss or security exposure.\n    \n* Do customers need to take any action?\n\n    * No. All queued work has been safely processed. If anything still looks off, please let us know and we will investigate immediately.\n    \n* How will we keep you updated?\n\n    * Your account team will share any follow-ups on improvements. We will also post future status updates through our standard channels if needed.","postmortem_body_last_updated_at":"2025-09-16T21:08:40.204Z","postmortem_ignored":false,"postmortem_published_at":"2025-09-16T21:08:40.204Z","postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"Application Not Loading","status":"resolved","created_at":"2025-09-09T19:55:12.650Z","updated_at":"2025-09-09T20:21:18.764Z","monitoring_at":null,"resolved_at":"2025-09-09T20:21:18.750Z","impact":"minor","shortlink":"https://stspg.io/jjry9dhbycfl","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-09-09T19:55:12.645Z","reminder_intervals":null,"id":"9g68zbgzqmp4","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"The issue preventing the app from loading has been resolved and functionality has been fully restored. We are continuing to monitor performance to ensure stability.","created_at":"2025-09-09T20:21:18.750Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-09T20:21:18.750Z","display_at":"2025-09-09T20:21:18.750Z","deliver_notifications":true,"tweet_id":null,"id":"5p7k457b0y0f","incident_id":"9g68zbgzqmp4","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"degraded_performance","new_status":"operational"}]},{"status":"investigating","body":"We’re currently investigating an issue where the app is not loading. Our engineering team is actively working to identify the cause and restore service as quickly as possible.","created_at":"2025-09-09T19:55:12.732Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-09-09T19:55:12.732Z","display_at":"2025-09-09T19:55:12.732Z","deliver_notifications":true,"tweet_id":null,"id":"st6pbvlg8yt9","incident_id":"9g68zbgzqmp4","custom_tweet":null,"affected_components":[{"code":"t36znl08dzw1","name":"Web App","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web App","created_at":"2023-12-14T19:37:58.546Z","updated_at":"2026-05-02T16:32:07.060Z","position":1,"description":null,"showcase":true,"start_date":"2023-12-14T00:00:00.000Z","id":"t36znl08dzw1","page_id":"xg1sy4d9c1r0","group_id":null}]},{"name":"Partial Outage","status":"resolved","created_at":"2025-08-28T18:30:00.000Z","updated_at":"2025-08-28T18:36:59.934Z","monitoring_at":null,"resolved_at":"2025-08-28T18:30:00.000Z","impact":"none","shortlink":"https://stspg.io/nfnyyhg0jx2g","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-08-28T18:36:34.281Z","reminder_intervals":null,"id":"gxpv3yl53ljg","page_id":"xg1sy4d9c1r0","incident_updates":[{"status":"resolved","body":"A brief performance degradation was observed. The issue caused slower load times and intermittent errors.","created_at":"2025-08-28T18:36:34.337Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-08-28T18:36:34.337Z","display_at":"2025-08-28T18:30:00.000Z","deliver_notifications":false,"tweet_id":null,"id":"d5cg8wzcg39p","incident_id":"gxpv3yl53ljg","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]}]
}