#!/usr/bin/env node

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

const fs = require("fs");
const path = require("path");
const { Worker } = require("worker_threads");
const os = require("os");

const MAX_WORKERS = Math.min(32, os.cpus().length);

const TASKCLUSTER_BASE_URL =
  process.env.TASKCLUSTER_PROXY_URL ||
  process.env.TASKCLUSTER_ROOT_URL ||
  "https://firefox-ci-tc.services.mozilla.com";

// Check for --harness parameter
const HARNESS = (() => {
  const harnessIndex = process.argv.findIndex(arg => arg === "--harness");
  if (harnessIndex !== -1 && harnessIndex + 1 < process.argv.length) {
    return process.argv[harnessIndex + 1];
  }
  return "xpcshell";
})();

// Firefox-CI ETL Query for test job data (contains xpcshell, mochitest, reftest)
const FIREFOX_CI_ETL_URL =
  "https://sql.telemetry.mozilla.org/api/queries/114029/results.json?api_key=6LTIeXwlJ5YTlmtbRXmlr5vfSEKVmzsNEyhr4VxO";

// Treeherder query for list of tasks to ignore (broken patches that were reverted)
const IGNORE_LIST_URL =
  "https://sql.telemetry.mozilla.org/api/queries/114030/results.json?api_key=8Q6UgAs8l8MdhmZD8bmW9VNcWpZ8MMwyhyOchslh";

// Check for --output-dir parameter
const OUTPUT_DIR = (() => {
  const outputDirIndex = process.argv.findIndex(arg => arg === "--output-dir");
  if (outputDirIndex !== -1 && outputDirIndex + 1 < process.argv.length) {
    return process.argv[outputDirIndex + 1];
  }
  return `./${HARNESS}-data`;
})();

const PROFILE_CACHE_DIR = "./profile-cache";

let previousRunData = null;
let allJobsCache = null;
let ignoreTasksCache = null;
let componentsData = null;
let dailyStatsMap = new Map();

const MOCHITEST_FLAVOR_PREFIXES = [
  ["devtools", "devtools"],
  ["browser", "browser-chrome"],
  ["chrome", "chrome"],
  ["a11y", "a11y"],
  ["plain", "plain"],
  ["media", "media"],
  ["remote", "remote"],
  ["webgl", "webgl"],
];

function classifyMochitestFlavor(jobName) {
  const m = jobName.match(/mochitest-(\w+)/);
  if (m) {
    for (const [prefix, flavor] of MOCHITEST_FLAVOR_PREFIXES) {
      if (m[1].startsWith(prefix)) {
        return flavor;
      }
    }
  }
  return "other";
}

if (!fs.existsSync(OUTPUT_DIR)) {
  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
}
if (!fs.existsSync(PROFILE_CACHE_DIR)) {
  fs.mkdirSync(PROFILE_CACHE_DIR, { recursive: true });
}

// Get date in YYYY-MM-DD format
function getDateString(daysAgo = 0) {
  const date = new Date();
  date.setDate(date.getDate() - daysAgo);
  return date.toISOString().split("T")[0];
}

// `optional` is for artifacts that may legitimately not exist, eg. a file that
// was added after the previous run generated its artifacts: a miss is expected
// and shouldn't be reported as an error.
async function fetchJson(url, optional = false) {
  const response = await fetch(url);
  if (!response.ok) {
    if (!optional) {
      console.error(
        `Failed to fetch ${url}: HTTP ${response.status} ${response.statusText}`
      );
    }
    return null;
  }
  return response.json();
}

// Fetch commit push data from Treeherder API
async function fetchCommitData(project, revision) {
  console.log(`Fetching commit data for ${project}:${revision}...`);

  const result = await fetchJson(
    `https://treeherder.mozilla.org/api/project/${project}/push/?full=true&count=10&revision=${revision}`
  );

  if (!result || !result.results || result.results.length === 0) {
    throw new Error(
      `No push found for revision ${revision} on project ${project}`
    );
  }

  const pushId = result.results[0].id;
  console.log(`Found push ID: ${pushId}`);
  return pushId;
}

// Fetch jobs from push
async function fetchPushJobs(project, pushId) {
  console.log(`Fetching jobs for push ID ${pushId}...`);

  let allJobs = [];
  let propertyNames = [];
  let url = `https://treeherder.mozilla.org/api/jobs/?push_id=${pushId}`;

  // The /jobs/ API is paginated, keep fetching until next is null
  while (url) {
    const result = await fetchJson(url);
    if (!result) {
      throw new Error(`Failed to fetch jobs for push ID ${pushId}`);
    }

    allJobs = allJobs.concat(result.results || []);
    if (!propertyNames.length) {
      propertyNames = result.job_property_names || [];
    }

    url = result.next;
  }

  // Get field indices dynamically
  const jobTypeNameIndex = propertyNames.indexOf("job_type_name");
  const taskIdIndex = propertyNames.indexOf("task_id");
  const retryIdIndex = propertyNames.indexOf("retry_id");
  const lastModifiedIndex = propertyNames.indexOf("last_modified");

  const harnessJobs = allJobs
    .filter(
      job => job[jobTypeNameIndex] && job[jobTypeNameIndex].includes(HARNESS)
    )
    .map(job => {
      const taskId = job[taskIdIndex];
      const retryId = job[retryIdIndex] || 0;
      const task = retryId === 0 ? taskId : `${taskId}.${retryId}`;
      return {
        name: job[jobTypeNameIndex],
        task,
        start_time: job[lastModifiedIndex],
        repository: project,
      };
    });

  console.log(
    `Found ${harnessJobs.length} ${HARNESS} jobs out of ${allJobs.length} total jobs`
  );
  return harnessJobs;
}

// Fetch test data from Firefox-CI ETL for a specific date
async function fetchHarnessData(targetDate) {
  console.log(`Fetching ${HARNESS} test data for ${targetDate}...`);

  // Fetch data from Firefox-CI ETL if not already cached
  if (!allJobsCache || !ignoreTasksCache) {
    console.log(`Querying Firefox-CI ETL and loading ignore list...`);

    // Fetch both Firefox-CI ETL data and ignore list in parallel
    const [etlResult, ignoreListResult] = await Promise.all([
      fetchJson(FIREFOX_CI_ETL_URL),
      fetchJson(IGNORE_LIST_URL),
    ]);

    if (!etlResult) {
      throw new Error("Failed to fetch data from Firefox-CI ETL");
    }

    if (!ignoreListResult) {
      throw new Error("Failed to fetch ignore list from Treeherder");
    }

    // Build set of tasks to ignore
    ignoreTasksCache = new Set();
    for (const row of ignoreListResult.query_result.data.rows) {
      ignoreTasksCache.add(row.task);
    }
    console.log(`Loaded ${ignoreTasksCache.size} tasks to ignore`);

    const allJobs = etlResult.query_result.data.rows;

    // Cache all harness jobs (don't filter by ignore list yet)
    allJobsCache = allJobs.filter(job => job.name?.includes(HARNESS));

    console.log(
      `Cached ${allJobsCache.length} ${HARNESS} jobs from Firefox-CI ETL (out of ${allJobs.length} total jobs)`
    );
  }

  // Filter cached jobs for the target date
  return allJobsCache.filter(job => job.start_time.startsWith(targetDate));
}

// Process jobs using worker threads with dynamic job distribution
async function processJobsWithWorkers(
  jobs,
  targetDate = null,
  onJobResult = null
) {
  if (jobs.length === 0) {
    return [];
  }

  const dateStr = targetDate ? ` for ${targetDate}` : "";
  console.log(
    `Processing ${jobs.length} jobs${dateStr} using ${MAX_WORKERS} workers...`
  );

  const jobQueue = [...jobs];
  const results = [];
  let invalidJobCount = 0;
  const workers = [];
  let completedJobs = 0;
  let lastProgressTime = 0;

  return new Promise((resolve, reject) => {
    // Track worker states
    const workerStates = new Map();

    // Create workers
    for (let i = 0; i < MAX_WORKERS; i++) {
      const worker = new Worker(path.join(__dirname, "profile-worker.js"), {
        workerData: {
          profileCacheDir: PROFILE_CACHE_DIR,
          taskclusterBaseUrl: TASKCLUSTER_BASE_URL,
        },
      });

      workers.push(worker);
      workerStates.set(worker, { id: i + 1, ready: false, jobsProcessed: 0 });

      worker.on("message", message => {
        const workerState = workerStates.get(worker);

        if (message.type === "ready") {
          workerState.ready = true;
          assignNextJob(worker);
        } else if (message.type === "jobComplete") {
          workerState.jobsProcessed++;
          completedJobs++;

          if (message.result) {
            if (message.result.error) {
              // Only network_error is retryable, permanent errors count as invalid
              if (message.result.error !== "network_error") {
                invalidJobCount++;
              }
            } else {
              // Let the caller fold large per-job data (markers) into a
              // running accumulator and free it, so the main thread doesn't
              // hold every job's raw markers at once.
              if (onJobResult) {
                onJobResult(message.result);
              }
              results.push(message.result);
            }
          }

          // Show progress at most once per second, or on first/last job
          const now = Date.now();
          if (
            completedJobs === 1 ||
            completedJobs === jobs.length ||
            now - lastProgressTime >= 1000
          ) {
            const percentage = Math.round((completedJobs / jobs.length) * 100);
            const paddedCompleted = completedJobs
              .toString()
              .padStart(jobs.length.toString().length);
            const paddedPercentage = percentage.toString().padStart(3); // Pad to 3 chars for alignment (0-100%)
            console.log(
              ` ${paddedPercentage}% ${paddedCompleted}/${jobs.length}`
            );
            lastProgressTime = now;
          }

          // Assign next job or finish
          assignNextJob(worker);
        } else if (message.type === "finished") {
          checkAllComplete();
        } else if (message.type === "error") {
          reject(new Error(`Worker ${workerState.id} error: ${message.error}`));
        }
      });

      worker.on("error", error => {
        reject(
          new Error(
            `Worker ${workerStates.get(worker).id} thread error: ${error.message}`
          )
        );
      });

      worker.on("exit", code => {
        if (code !== 0) {
          reject(
            new Error(
              `Worker ${workerStates.get(worker).id} stopped with exit code ${code}`
            )
          );
        }
      });
    }

    function assignNextJob(worker) {
      if (jobQueue.length) {
        const job = jobQueue.shift();
        worker.postMessage({ type: "job", job });
      } else {
        // No more jobs, tell worker to finish
        worker.postMessage({ type: "shutdown" });
      }
    }

    let resolved = false;
    let workersFinished = 0;

    function checkAllComplete() {
      if (resolved) {
        return;
      }

      workersFinished++;

      if (workersFinished >= MAX_WORKERS) {
        resolved = true;

        // Terminate all workers to ensure clean exit
        workers.forEach(worker => worker.terminate());

        resolve({ results, invalidJobCount });
      }
    }
  });
}

// Fetch Bugzilla component mapping data
async function fetchComponentsData() {
  if (componentsData) {
    return componentsData;
  }

  console.log("Fetching Bugzilla component mapping...");
  const url = `${TASKCLUSTER_BASE_URL}/api/index/v1/task/gecko.v2.mozilla-central.latest.source.source-bugzilla-info/artifacts/public/components-normalized.json`;

  try {
    componentsData = await fetchJson(url);
    console.log("Component mapping loaded successfully");
    return componentsData;
  } catch (error) {
    console.error("Failed to fetch component mapping:", error);
    return null;
  }
}

// Look up component for a test path
function findComponentForPath(testPath) {
  if (!componentsData || !componentsData.paths) {
    return null;
  }

  const parts = testPath.split("/");
  let current = componentsData.paths;

  for (const part of parts) {
    if (typeof current === "number") {
      return current;
    }
    if (typeof current === "object" && current !== null && part in current) {
      current = current[part];
    } else {
      return null;
    }
  }

  return typeof current === "number" ? current : null;
}

// Get component string from component ID
function getComponentString(componentId) {
  if (!componentsData || !componentsData.components || componentId == null) {
    return null;
  }

  const component = componentsData.components[String(componentId)];
  if (!component || !Array.isArray(component) || component.length !== 2) {
    return null;
  }

  return `${component[0]} :: ${component[1]}`;
}

// Helper function to determine if a status should include message data
function shouldIncludeMessage(status) {
  return status === "SKIP" || status.startsWith("FAIL");
}

// Create the building blocks of our columnar JSON formats: the string tables and
// the interning primitive that keeps them deduplicated, plus the taskInfo and
// testInfo side tables describing the tasks and tests the file refers to.
// `extraTableNames` are the string tables specific to one format.
function createColumnarTables(extraTableNames) {
  const tables = {};
  const stringMaps = {};
  for (const tableName of [
    "jobNames",
    "testPaths",
    "testNames",
    "repositories",
    "taskIds",
    "components",
    "commitIds",
    ...extraTableNames,
  ]) {
    tables[tableName] = [];
    stringMaps[tableName] = new Map();
  }

  // Intern a string into one of the tables, returning its index, or null for a
  // null/undefined value. Empty strings are interned like any other value.
  function internString(tableName, value) {
    if (value === null || value === undefined) {
      return null;
    }
    const map = stringMaps[tableName];
    let index = map.get(value);
    if (index === undefined) {
      index = tables[tableName].length;
      tables[tableName].push(value);
      map.set(value, index);
    }
    return index;
  }

  function componentIdForPath(filePath) {
    const componentString = getComponentString(findComponentForPath(filePath));
    return componentString ? internString("components", componentString) : null;
  }

  // Parallel arrays indexed by taskIdId.
  const taskInfo = {
    repositoryIds: [],
    jobNameIds: [],
    commitIds: [],
  };

  // Intern the task a job result comes from, recording the task's repository,
  // job name and commit the first time we see it.
  function getTaskIdId(result) {
    const taskIdId = internString(
      "taskIds",
      `${result.taskId}.${result.retryId}`
    );
    if (taskInfo.repositoryIds[taskIdId] === undefined) {
      taskInfo.repositoryIds[taskIdId] = internString(
        "repositories",
        result.repository
      );
      taskInfo.jobNameIds[taskIdId] = internString("jobNames", result.jobName);
      taskInfo.commitIds[taskIdId] = internString("commitIds", result.commitId);
    }
    return taskIdId;
  }

  // Parallel arrays indexed by testId.
  const testInfo = {
    testPathIds: [],
    testNameIds: [],
    componentIds: [],
  };
  const testIds = new Map();

  // Intern a test by its full path, split into a directory and a file name, and
  // look up its Bugzilla component.
  function getTestId(fullPath) {
    let testId = testIds.get(fullPath);
    if (testId !== undefined) {
      return testId;
    }

    const lastSlashIndex = fullPath.lastIndexOf("/");
    let testPath, testName;
    if (lastSlashIndex === -1) {
      testPath = "";
      testName = fullPath;
    } else {
      testPath = fullPath.substring(0, lastSlashIndex);
      testName = fullPath.substring(lastSlashIndex + 1);
    }

    testId = testInfo.testPathIds.length;
    testInfo.testPathIds.push(internString("testPaths", testPath));
    testInfo.testNameIds.push(internString("testNames", testName));
    testInfo.componentIds.push(componentIdForPath(fullPath));
    testIds.set(fullPath, testId);
    return testId;
  }

  return {
    tables,
    internString,
    componentIdForPath,
    taskInfo,
    getTaskIdId,
    testInfo,
    getTestId,
  };
}

// Zeroed reference counts for each string table, for the caller to fill in
// before calling sortTablesByFrequency.
function createFrequencyCounts(tables) {
  const frequencyCounts = {};
  for (const [tableName, table] of Object.entries(tables)) {
    frequencyCounts[tableName] = new Array(table.length).fill(0);
  }
  return frequencyCounts;
}

// Sort each string table by how often its entries are referenced and drop the
// unreferenced ones, so that the most frequent strings get the smallest indices
// and the file compresses better. Returns the sorted tables and, for each table,
// a map from old to new index.
function sortTablesByFrequency(tables, frequencyCounts) {
  const sortedTables = {};
  const indexMaps = {};

  for (const [tableName, table] of Object.entries(tables)) {
    const counts = frequencyCounts[tableName];
    const sorted = table
      .map((value, oldIndex) => ({ value, oldIndex, count: counts[oldIndex] }))
      .filter(item => item.count > 0)
      .sort((a, b) => {
        if (b.count !== a.count) {
          return b.count - a.count;
        }
        // Codepoint order rather than localeCompare, which would sort with the
        // host's default locale and make the output machine dependent.
        return a.value < b.value ? -1 : 1;
      });

    sortedTables[tableName] = sorted.map(item => item.value);
    indexMaps[tableName] = new Map(
      sorted.map((item, newIndex) => [item.oldIndex, newIndex])
    );
  }

  return { sortedTables, indexMaps };
}

// taskInfo's arrays are indexed by taskIdId, so remapping the taskIds table
// means rebuilding them at the new indices.
function remapTaskInfo(taskInfo, indexMaps) {
  const sortedTaskInfo = {
    repositoryIds: [],
    jobNameIds: [],
    commitIds: [],
  };
  const hasChunks = !!taskInfo.chunks;
  if (hasChunks) {
    sortedTaskInfo.chunks = [];
  }

  for (
    let oldTaskIdId = 0;
    oldTaskIdId < taskInfo.repositoryIds.length;
    oldTaskIdId++
  ) {
    const newTaskIdId = indexMaps.taskIds.get(oldTaskIdId);
    if (newTaskIdId === undefined) {
      continue;
    }
    sortedTaskInfo.repositoryIds[newTaskIdId] = indexMaps.repositories.get(
      taskInfo.repositoryIds[oldTaskIdId]
    );
    sortedTaskInfo.jobNameIds[newTaskIdId] = indexMaps.jobNames.get(
      taskInfo.jobNameIds[oldTaskIdId]
    );
    sortedTaskInfo.commitIds[newTaskIdId] =
      taskInfo.commitIds[oldTaskIdId] === null
        ? null
        : indexMaps.commitIds.get(taskInfo.commitIds[oldTaskIdId]);
    if (hasChunks) {
      sortedTaskInfo.chunks[newTaskIdId] = taskInfo.chunks[oldTaskIdId] ?? null;
    }
  }

  return sortedTaskInfo;
}

function remapTestInfo(testInfo, indexMaps) {
  return {
    testPathIds: testInfo.testPathIds.map(oldId =>
      indexMaps.testPaths.get(oldId)
    ),
    testNameIds: testInfo.testNameIds.map(oldId =>
      indexMaps.testNames.get(oldId)
    ),
    componentIds: testInfo.componentIds.map(oldId =>
      oldId === null ? null : indexMaps.components.get(oldId)
    ),
  };
}

// Create string tables and store raw data efficiently
function createDataTables(jobResults) {
  const { tables, internString, taskInfo, getTaskIdId, testInfo, getTestId } =
    createColumnarTables(["statuses", "messages", "crashSignatures"]);

  // Test runs grouped by test ID, then by status ID
  // testRuns[testId] = array of status groups for that test
  const testRuns = [];

  for (const result of jobResults) {
    if (!result || !result.timings) {
      continue;
    }

    const taskIdId = getTaskIdId(result);

    for (const timing of result.timings) {
      const testId = getTestId(timing.path);
      const statusId = internString("statuses", timing.status || "UNKNOWN");

      // Initialize test group if it doesn't exist
      if (!testRuns[testId]) {
        testRuns[testId] = [];
      }

      // Initialize status group within test if it doesn't exist
      let statusGroup = testRuns[testId][statusId];
      if (!statusGroup) {
        statusGroup = {
          taskIdIds: [],
          durations: [],
          timestamps: [],
        };
        // Include messageIds array for statuses that should have messages
        if (shouldIncludeMessage(timing.status)) {
          statusGroup.messageIds = [];
        }
        // Only include crash data arrays for CRASH status
        if (timing.status === "CRASH") {
          statusGroup.crashSignatureIds = [];
          statusGroup.minidumps = [];
        }
        testRuns[testId][statusId] = statusGroup;
      }

      // Add test run to the appropriate test/status group
      statusGroup.taskIdIds.push(taskIdId);
      statusGroup.durations.push(Math.round(timing.duration));
      statusGroup.timestamps.push(timing.timestamp);

      // Store message ID for statuses that should include messages (or null if no message)
      if (shouldIncludeMessage(timing.status)) {
        statusGroup.messageIds.push(
          internString("messages", timing.message || null)
        );
      }

      // Store crash data for CRASH status (or null if not available)
      if (timing.status === "CRASH") {
        statusGroup.crashSignatureIds.push(
          internString("crashSignatures", timing.crashSignature || null)
        );
        statusGroup.minidumps.push(timing.minidump || null);
      }
    }
  }

  return {
    tables,
    taskInfo,
    testInfo,
    testRuns,
  };
}

// Sort string tables by frequency and remap all indices for deterministic output and better compression
function sortStringTablesByFrequency(dataStructure) {
  const { tables, taskInfo, testInfo, testRuns } = dataStructure;

  // Count frequency of each index for each table
  const frequencyCounts = createFrequencyCounts(tables);

  // Count taskInfo references
  for (const jobNameId of taskInfo.jobNameIds) {
    if (jobNameId !== undefined) {
      frequencyCounts.jobNames[jobNameId]++;
    }
  }
  for (const repositoryId of taskInfo.repositoryIds) {
    if (repositoryId !== undefined) {
      frequencyCounts.repositories[repositoryId]++;
    }
  }
  for (const commitId of taskInfo.commitIds) {
    if (commitId !== null) {
      frequencyCounts.commitIds[commitId]++;
    }
  }

  // Count testInfo references
  for (const testPathId of testInfo.testPathIds) {
    frequencyCounts.testPaths[testPathId]++;
  }
  for (const testNameId of testInfo.testNameIds) {
    frequencyCounts.testNames[testNameId]++;
  }
  for (const componentId of testInfo.componentIds) {
    if (componentId !== null) {
      frequencyCounts.components[componentId]++;
    }
  }

  // Count testRuns references
  for (const testGroup of testRuns) {
    if (!testGroup) {
      continue;
    }

    testGroup.forEach((statusGroup, statusId) => {
      if (!statusGroup) {
        return;
      }

      // Handle aggregated format (counts/days), bucket format (durations),
      // and detailed format (taskIdIds)
      if (statusGroup.taskIdIds) {
        // Check if taskIdIds is array of arrays (aggregated) or flat array (daily)
        const isArrayOfArrays =
          !!statusGroup.taskIdIds.length &&
          Array.isArray(statusGroup.taskIdIds[0]);

        if (isArrayOfArrays) {
          // Aggregated format: array of arrays
          const totalRuns = statusGroup.taskIdIds.reduce(
            (sum, arr) => sum + arr.length,
            0
          );
          frequencyCounts.statuses[statusId] += totalRuns;

          for (const taskIdIdsArray of statusGroup.taskIdIds) {
            for (const taskIdId of taskIdIdsArray) {
              frequencyCounts.taskIds[taskIdId]++;
            }
          }
        } else {
          // Daily format: flat array
          frequencyCounts.statuses[statusId] += statusGroup.taskIdIds.length;

          for (const taskIdId of statusGroup.taskIdIds) {
            frequencyCounts.taskIds[taskIdId]++;
          }
        }
      } else if (
        statusGroup.durations &&
        Array.isArray(statusGroup.durations[0])
      ) {
        // Bucket pass format: durations is array of arrays
        const totalRuns = statusGroup.durations.reduce(
          (sum, arr) => sum + arr.length,
          0
        );
        frequencyCounts.statuses[statusId] += totalRuns;
      } else if (statusGroup.counts) {
        // Aggregated passing tests - count total runs
        const totalRuns = statusGroup.counts.reduce((a, b) => a + b, 0);
        frequencyCounts.statuses[statusId] += totalRuns;
      }

      if (statusGroup.jobNameIds) {
        for (const jobNameId of statusGroup.jobNameIds) {
          if (jobNameId !== null) {
            frequencyCounts.jobNames[jobNameId]++;
          }
        }
      }

      if (statusGroup.messageIds) {
        for (const messageId of statusGroup.messageIds) {
          if (messageId !== null) {
            frequencyCounts.messages[messageId]++;
          }
        }
      }

      if (statusGroup.crashSignatureIds) {
        for (const crashSigId of statusGroup.crashSignatureIds) {
          if (crashSigId !== null) {
            frequencyCounts.crashSignatures[crashSigId]++;
          }
        }
      }
    });
  }

  const { sortedTables, indexMaps } = sortTablesByFrequency(
    tables,
    frequencyCounts
  );
  const sortedTaskInfo = remapTaskInfo(taskInfo, indexMaps);
  const sortedTestInfo = remapTestInfo(testInfo, indexMaps);

  // Remap testRuns indices
  const sortedTestRuns = testRuns.map(testGroup => {
    if (!testGroup) {
      return testGroup;
    }

    return testGroup.map(statusGroup => {
      if (!statusGroup) {
        return statusGroup;
      }

      // Bucket pass format: durations is array of arrays, with jobNameIds
      if (
        statusGroup.durations &&
        Array.isArray(statusGroup.durations[0]) &&
        !statusGroup.taskIdIds
      ) {
        const remapped = {
          durations: statusGroup.durations,
          days: statusGroup.days,
        };
        if (statusGroup.jobNameIds) {
          remapped.jobNameIds = statusGroup.jobNameIds.map(oldId =>
            oldId === null ? null : indexMaps.jobNames.get(oldId)
          );
        }
        return remapped;
      }

      // Aggregated counts format (may have jobNameIds/messageIds in bucket files)
      if (statusGroup.counts && !statusGroup.taskIdIds) {
        const remapped = {
          counts: statusGroup.counts,
          days: statusGroup.days,
        };
        if (statusGroup.jobNameIds) {
          remapped.jobNameIds = statusGroup.jobNameIds.map(oldId =>
            oldId === null ? null : indexMaps.jobNames.get(oldId)
          );
        }
        if (statusGroup.messageIds) {
          remapped.messageIds = statusGroup.messageIds.map(oldId =>
            oldId === null ? null : indexMaps.messages.get(oldId)
          );
        }
        return remapped;
      }

      // Check if this is aggregated format (array of arrays) or daily format (flat array)
      const isArrayOfArrays =
        !!statusGroup.taskIdIds.length &&
        Array.isArray(statusGroup.taskIdIds[0]);

      const remapped = {};

      if (isArrayOfArrays) {
        // Aggregated format: array of arrays with days
        remapped.taskIdIds = statusGroup.taskIdIds.map(taskIdIdsArray =>
          taskIdIdsArray.map(oldId => indexMaps.taskIds.get(oldId))
        );
        remapped.days = statusGroup.days;
      } else {
        // Daily format: flat array with durations and timestamps
        remapped.taskIdIds = statusGroup.taskIdIds.map(oldId =>
          indexMaps.taskIds.get(oldId)
        );
        remapped.durations = statusGroup.durations;
        remapped.timestamps = statusGroup.timestamps;
      }

      // Remap message IDs for status groups that have messages
      if (statusGroup.messageIds) {
        remapped.messageIds = statusGroup.messageIds.map(oldId =>
          oldId === null ? null : indexMaps.messages.get(oldId)
        );
      }

      // Remap crash data for CRASH status
      if (statusGroup.crashSignatureIds) {
        remapped.crashSignatureIds = statusGroup.crashSignatureIds.map(oldId =>
          oldId === null ? null : indexMaps.crashSignatures.get(oldId)
        );
      }
      if (statusGroup.minidumps) {
        remapped.minidumps = statusGroup.minidumps;
      }

      return remapped;
    });
  });

  // Remap statusId positions in testRuns (move status groups to their new positions)
  const finalTestRuns = sortedTestRuns.map(testGroup => {
    if (!testGroup) {
      return testGroup;
    }

    const remappedGroup = [];
    testGroup.forEach((statusGroup, oldStatusId) => {
      if (!statusGroup) {
        return;
      }
      const newStatusId = indexMaps.statuses.get(oldStatusId);
      remappedGroup[newStatusId] = statusGroup;
    });

    return remappedGroup;
  });

  return {
    tables: sortedTables,
    taskInfo: sortedTaskInfo,
    testInfo: sortedTestInfo,
    testRuns: finalTestRuns,
  };
}

// Create resource usage data structure
function createResourceUsageData(jobResults) {
  const jobNames = [];
  const jobNameMap = new Map();
  const repositories = [];
  const repositoryMap = new Map();
  const machineInfos = [];
  const machineInfoMap = new Map();

  // Collect all job data first
  const jobDataList = [];

  for (const result of jobResults) {
    if (!result || !result.resourceUsage) {
      continue;
    }

    // Extract chunk number from job name (e.g., "test-linux1804-64/opt-xpcshell-1" -> "test-linux1804-64/opt-xpcshell", chunk: 1)
    let jobNameBase = result.jobName;
    let chunkNumber = null;
    const match = result.jobName.match(/^(.+)-(\d+)$/);
    if (match) {
      jobNameBase = match[1];
      chunkNumber = parseInt(match[2], 10);
    }

    // Get or create job name index
    let jobNameId = jobNameMap.get(jobNameBase);
    if (jobNameId === undefined) {
      jobNameId = jobNames.length;
      jobNames.push(jobNameBase);
      jobNameMap.set(jobNameBase, jobNameId);
    }

    // Get or create repository index
    let repositoryId = repositoryMap.get(result.repository);
    if (repositoryId === undefined) {
      repositoryId = repositories.length;
      repositories.push(result.repository);
      repositoryMap.set(result.repository, repositoryId);
    }

    // Get or create machine info index
    const machineInfo = result.resourceUsage.machineInfo;
    const machineInfoKey = JSON.stringify(machineInfo);
    let machineInfoId = machineInfoMap.get(machineInfoKey);
    if (machineInfoId === undefined) {
      machineInfoId = machineInfos.length;
      machineInfos.push(machineInfo);
      machineInfoMap.set(machineInfoKey, machineInfoId);
    }

    // Combine taskId and retryId (omit .0 for retry 0)
    const taskIdString =
      result.retryId === 0
        ? result.taskId
        : `${result.taskId}.${result.retryId}`;

    jobDataList.push({
      jobNameId,
      chunk: chunkNumber,
      taskId: taskIdString,
      repositoryId,
      startTime: result.startTime,
      machineInfoId,
      maxMemory: result.resourceUsage.maxMemory,
      idleTime: result.resourceUsage.idleTime,
      singleCoreTime: result.resourceUsage.singleCoreTime,
      cpuBuckets: result.resourceUsage.cpuBuckets,
    });
  }

  // Sort by start time
  jobDataList.sort((a, b) => a.startTime - b.startTime);

  // Apply differential compression to start times and build parallel arrays
  const jobs = {
    jobNameIds: [],
    chunks: [],
    taskIds: [],
    repositoryIds: [],
    startTimes: [],
    machineInfoIds: [],
    maxMemories: [],
    idleTimes: [],
    singleCoreTimes: [],
    cpuBuckets: [],
  };

  let previousStartTime = 0;
  for (const jobData of jobDataList) {
    jobs.jobNameIds.push(jobData.jobNameId);
    jobs.chunks.push(jobData.chunk);
    jobs.taskIds.push(jobData.taskId);
    jobs.repositoryIds.push(jobData.repositoryId);

    // Differential compression: store difference from previous
    const timeDiff = jobData.startTime - previousStartTime;
    jobs.startTimes.push(timeDiff);
    previousStartTime = jobData.startTime;

    jobs.machineInfoIds.push(jobData.machineInfoId);
    jobs.maxMemories.push(jobData.maxMemory);
    jobs.idleTimes.push(jobData.idleTime);
    jobs.singleCoreTimes.push(jobData.singleCoreTime);
    jobs.cpuBuckets.push(jobData.cpuBuckets);
  }

  return {
    jobNames,
    repositories,
    machineInfos,
    jobs,
  };
}

// Create an incremental accumulator for error/warning marker data.
//
// The data is fully columnar: on top of the shared string tables, the `messages`
// table interns each unique (marker name, message text, file, line, component)
// so a line and file are stored once per distinct message rather than once per
// occurrence. Occurrences are grouped by (test, message) into `markers`, with
// per-group taskIdIds/counts sub-arrays, so the test and message are stored once
// per group rather than once per (test, task, message) occurrence. This supports
// aggregating by test, by message, and by component, plus per-task drill-down,
// while keeping the on-disk size small.
//
// `feedJob(result)` folds one job's markers in (so the caller can free the raw
// markers and avoid holding every job's markers at once); `finalize()` returns
// the assembled { tables, messages, taskInfo, testInfo, markers } structure.
function createMarkerAccumulator() {
  const {
    tables,
    internString,
    componentIdForPath,
    taskInfo,
    getTaskIdId,
    testInfo,
    getTestId,
  } = createColumnarTables(["markerNames", "messageTexts", "files"]);

  // Interned (marker name, message text, file, line, component) table, indexed
  // by messageId. The marker name lives here because it is intrinsic to the
  // diagnostic, not to each occurrence.
  const messages = {
    markerNameIds: [],
    textIds: [],
    fileIds: [],
    lines: [],
    componentIds: [],
  };
  const messageIds = new Map();

  function internMessage(markerNameId, textId, fileId, line, componentId) {
    const key = `${markerNameId}|${textId}|${fileId}|${line}|${componentId}`;
    let index = messageIds.get(key);
    if (index === undefined) {
      index = messages.markerNameIds.length;
      messages.markerNameIds.push(markerNameId);
      messages.textIds.push(textId);
      messages.fileIds.push(fileId);
      messages.lines.push(line);
      messages.componentIds.push(componentId);
      messageIds.set(key, index);
    }
    return index;
  }

  // Occurrences accumulated as a nested map: testId -> messageId -> taskIdId ->
  // count. Grouping by (test, message) as markers arrive means finalize() can
  // emit the grouped columnar form directly, without ever holding one row per
  // (test, task, message) occurrence (there are tens of millions of those in a
  // day, but far fewer distinct (test, message) groups).
  const groups = new Map();

  // Cache the component lookup, which walks the component tree and is done for
  // every occurrence: a day's markers come from far fewer distinct files than
  // there are occurrences.
  const fileComponentIds = new Map();
  function componentIdForFile(file) {
    if (!fileComponentIds.has(file)) {
      fileComponentIds.set(file, file ? componentIdForPath(file) : null);
    }
    return fileComponentIds.get(file);
  }

  function getMessageId(markerNameId, message, file, line, testId) {
    return internMessage(
      markerNameId,
      internString("messageTexts", message),
      internString("files", file),
      line,
      // Attribute the message to the component of its source file, falling back
      // to the component of the running test when the file maps to no component
      // (console.* markers have no file at all).
      componentIdForFile(file) ?? testInfo.componentIds[testId]
    );
  }

  function feedJob(result) {
    if (!result.markers.length) {
      return;
    }

    const taskIdId = getTaskIdId(result);

    for (const marker of result.markers) {
      // The test that was running when the marker was emitted (empty when the
      // marker fired outside of any test, e.g. during setup/shutdown).
      const testId = getTestId(marker.test || "");
      const messageId = getMessageId(
        internString("markerNames", marker.name),
        marker.message,
        marker.file,
        marker.line,
        testId
      );

      let byMessage = groups.get(testId);
      if (byMessage === undefined) {
        byMessage = new Map();
        groups.set(testId, byMessage);
      }
      let byTask = byMessage.get(messageId);
      if (byTask === undefined) {
        byTask = new Map();
        byMessage.set(messageId, byTask);
      }
      byTask.set(taskIdId, (byTask.get(taskIdId) || 0) + 1);
    }
  }

  // Emit the grouped columnar form: one entry per (test, message) group, with
  // parallel taskIdIds/counts sub-arrays. Indices are still in interning order;
  // sortMarkerStringTables frequency-sorts and orders them. The nested map is
  // drained as it is walked so its (large) inner maps can be reclaimed.
  function finalize() {
    const markers = {
      testIds: [],
      messageIds: [],
      taskIdIds: [],
      counts: [],
    };
    for (const [testId, byMessage] of groups) {
      for (const [messageId, byTask] of byMessage) {
        const taskIdIds = [];
        const counts = [];
        for (const [taskIdId, count] of byTask) {
          taskIdIds.push(taskIdId);
          counts.push(count);
        }
        markers.testIds.push(testId);
        markers.messageIds.push(messageId);
        markers.taskIdIds.push(taskIdIds);
        markers.counts.push(counts);
      }
      groups.delete(testId);
    }

    return { tables, messages, taskInfo, testInfo, markers };
  }

  return { feedJob, finalize };
}

// Sort the marker string tables and the message table by frequency and remap all
// indices, so that the most frequent values get the smallest indices and the file
// compresses better. Operates on the structure produced by
// createMarkerAccumulator.
function sortMarkerStringTables(dataStructure) {
  const { tables, messages, taskInfo, testInfo, markers } = dataStructure;

  const frequencyCounts = createFrequencyCounts(tables);

  for (const jobNameId of taskInfo.jobNameIds) {
    frequencyCounts.jobNames[jobNameId]++;
  }
  for (const repositoryId of taskInfo.repositoryIds) {
    frequencyCounts.repositories[repositoryId]++;
  }
  for (const commitId of taskInfo.commitIds) {
    if (commitId !== null) {
      frequencyCounts.commitIds[commitId]++;
    }
  }

  for (const testPathId of testInfo.testPathIds) {
    frequencyCounts.testPaths[testPathId]++;
  }
  for (const testNameId of testInfo.testNameIds) {
    frequencyCounts.testNames[testNameId]++;
  }
  for (const componentId of testInfo.componentIds) {
    if (componentId !== null) {
      frequencyCounts.components[componentId]++;
    }
  }

  for (const markerNameId of messages.markerNameIds) {
    frequencyCounts.markerNames[markerNameId]++;
  }
  for (const textId of messages.textIds) {
    if (textId !== null) {
      frequencyCounts.messageTexts[textId]++;
    }
  }
  for (const fileId of messages.fileIds) {
    if (fileId !== null) {
      frequencyCounts.files[fileId]++;
    }
  }
  for (const componentId of messages.componentIds) {
    if (componentId !== null) {
      frequencyCounts.components[componentId]++;
    }
  }

  for (const groupTasks of markers.taskIdIds) {
    for (const taskIdId of groupTasks) {
      frequencyCounts.taskIds[taskIdId]++;
    }
  }

  const { sortedTables, indexMaps } = sortTablesByFrequency(
    tables,
    frequencyCounts
  );

  // Remap the message table's string indices, then reorder its rows by how often
  // occurrences reference them (most-referenced first), so that the messageId
  // values stored per occurrence are small.
  const remappedMarkerNameIds = messages.markerNameIds.map(id =>
    indexMaps.markerNames.get(id)
  );
  const remappedTextIds = messages.textIds.map(id =>
    id === null ? null : indexMaps.messageTexts.get(id)
  );
  const remappedFileIds = messages.fileIds.map(id =>
    id === null ? null : indexMaps.files.get(id)
  );
  const remappedMessageComponentIds = messages.componentIds.map(id =>
    id === null ? null : indexMaps.components.get(id)
  );

  const messageRefCounts = new Array(messages.markerNameIds.length).fill(0);
  for (let g = 0; g < markers.messageIds.length; g++) {
    messageRefCounts[markers.messageIds[g]] += markers.taskIdIds[g].length;
  }
  const messageOrder = Array.from(
    { length: messages.markerNameIds.length },
    (_, i) => i
  ).sort((a, b) => messageRefCounts[b] - messageRefCounts[a] || a - b);

  const messageRowMap = new Map();
  messageOrder.forEach((oldId, newId) => messageRowMap.set(oldId, newId));

  const sortedMessages = {
    markerNameIds: messageOrder.map(i => remappedMarkerNameIds[i]),
    textIds: messageOrder.map(i => remappedTextIds[i]),
    fileIds: messageOrder.map(i => remappedFileIds[i]),
    lines: messageOrder.map(i => messages.lines[i]),
    componentIds: messageOrder.map(i => remappedMessageComponentIds[i]),
  };

  // Remap and reorder the grouped occurrences. testIds index into testInfo,
  // whose row order is unchanged, so they need no remap; messageIds are remapped
  // through messageRowMap (message rows were reordered by reference frequency);
  // taskIdIds index into tables.taskIds (reordered by frequency). Within each
  // group the tasks are sorted ascending (carrying their counts) and then
  // delta-encoded on the way out, so the stored deltas are small positive
  // integers. Groups are ordered by (test, message).
  const { testIds } = markers;
  const groupCount = testIds.length;

  const newMessageIds = markers.messageIds.map(id => messageRowMap.get(id));
  const sortedTaskIdIds = new Array(groupCount);
  const sortedCounts = new Array(groupCount);
  for (let g = 0; g < groupCount; g++) {
    const tasks = markers.taskIdIds[g];
    const groupCounts = markers.counts[g];
    const remapped = tasks.map((id, j) => [
      indexMaps.taskIds.get(id),
      groupCounts[j],
    ]);
    remapped.sort((a, b) => a[0] - b[0]);
    let prev = 0;
    const deltas = new Array(remapped.length);
    const counts = new Array(remapped.length);
    for (let j = 0; j < remapped.length; j++) {
      deltas[j] = remapped[j][0] - prev;
      prev = remapped[j][0];
      counts[j] = remapped[j][1];
    }
    sortedTaskIdIds[g] = deltas;
    sortedCounts[g] = counts;
  }

  const order = Array.from({ length: groupCount }, (_, i) => i);
  order.sort(
    (a, b) => testIds[a] - testIds[b] || newMessageIds[a] - newMessageIds[b]
  );

  return {
    tables: sortedTables,
    messages: sortedMessages,
    taskInfo: remapTaskInfo(taskInfo, indexMaps),
    testInfo: remapTestInfo(testInfo, indexMaps),
    markers: {
      testIds: order.map(i => testIds[i]),
      messageIds: order.map(i => newMessageIds[i]),
      taskIdIds: order.map(i => sortedTaskIdIds[i]),
      counts: order.map(i => sortedCounts[i]),
    },
  };
}

// Total the occurrences of each marker name (markers.counts holds one array of
// per-task counts per group), eg. { "C++ warning": 8137, "console.error": 12 }.
function markerOccurrencesByName({ tables, messages, markers }) {
  const occurrences = {};
  for (let g = 0; g < markers.messageIds.length; g++) {
    const nameId = messages.markerNameIds[markers.messageIds[g]];
    const name = tables.markerNames[nameId];
    for (const count of markers.counts[g]) {
      occurrences[name] = (occurrences[name] || 0) + count;
    }
  }
  return occurrences;
}

// Save the error/warning marker file, if we managed to build it, and report
// whether we now have that file. This is the largest of our outputs and the least
// critical one, and JSON.stringify has a string length limit that a mochitest day
// can reach, so a failure here is logged and otherwise ignored rather than costing
// us the day's timings and stats.
function saveMarkerData(markerData, filePath) {
  if (!markerData) {
    return false;
  }

  try {
    saveJsonFile(markerData, filePath);
  } catch (error) {
    console.error(`Error saving ${filePath}:`, error);
    return false;
  }

  return true;
}

// Helper to save a JSON file and log its size
function saveJsonFile(data, filePath) {
  fs.writeFileSync(filePath, JSON.stringify(data));

  const stats = fs.statSync(filePath);
  const fileSizeBytes = stats.size;

  // Use MB for files >= 1MB, otherwise KB
  if (fileSizeBytes >= 1024 * 1024) {
    const fileSizeMB = Math.round(fileSizeBytes / (1024 * 1024));
    const formattedBytes = fileSizeBytes.toLocaleString();
    console.log(
      `Saved ${filePath} - ${fileSizeMB}MB (${formattedBytes} bytes)`
    );
  } else {
    const fileSizeKB = Math.round(fileSizeBytes / 1024);
    console.log(`Saved ${filePath} - ${fileSizeKB}KB`);
  }
}

// Common function to process jobs and create data structure
async function processJobsAndCreateData(
  jobs,
  targetLabel,
  startTime,
  metadata
) {
  if (jobs.length === 0) {
    console.log(`No jobs found for ${targetLabel}.`);
    return null;
  }

  // Process jobs to extract test timings. Fold each job's error/warning markers
  // into the accumulator as it arrives and free the raw markers, so the main
  // thread never holds every job's (potentially millions of) markers at once.
  // The marker data is a secondary output: if building it fails we give up on it
  // and still produce the day's timings, resource usage and stats.
  let markerAccumulator = createMarkerAccumulator();
  const jobProcessingStart = Date.now();
  const { results: jobResults, invalidJobCount } = await processJobsWithWorkers(
    jobs,
    targetLabel,
    result => {
      try {
        markerAccumulator?.feedJob(result);
      } catch (error) {
        console.error("Error accumulating error/warning markers:", error);
        markerAccumulator = null;
      }
      delete result.markers;
    }
  );
  const jobProcessingTime = Date.now() - jobProcessingStart;
  console.log(
    `Successfully processed ${jobResults.length} jobs in ${jobProcessingTime}ms`
  );

  // Create efficient data tables
  const dataTablesStart = Date.now();
  let dataStructure = createDataTables(jobResults);
  const dataTablesTime = Date.now() - dataTablesStart;
  console.log(`Created data tables in ${dataTablesTime}ms:`);

  // Check if any test runs were extracted
  const hasTestRuns = !!dataStructure.testRuns.length;
  if (!hasTestRuns) {
    console.log(`No test run data extracted for ${targetLabel}`);
    return null;
  }

  const totalRuns = dataStructure.testRuns.reduce((sum, testGroup) => {
    if (!testGroup) {
      return sum;
    }
    return (
      sum +
      testGroup.reduce(
        (testSum, statusGroup) =>
          testSum + (statusGroup ? statusGroup.taskIdIds.length : 0),
        0
      )
    );
  }, 0);
  console.log(
    `  ${dataStructure.testInfo.testPathIds.length} tests, ${totalRuns} runs, ${dataStructure.tables.taskIds.length} tasks, ${dataStructure.tables.jobNames.length} job names, ${dataStructure.tables.statuses.length} statuses`
  );

  // Sort string tables by frequency for deterministic output and better compression
  const sortingStart = Date.now();
  dataStructure = sortStringTablesByFrequency(dataStructure);
  const sortingTime = Date.now() - sortingStart;
  console.log(`Sorted string tables by frequency in ${sortingTime}ms`);

  // Convert absolute timestamps to relative and apply differential compression (in place)
  for (const testGroup of dataStructure.testRuns) {
    if (!testGroup) {
      continue;
    }

    for (const statusGroup of testGroup) {
      if (!statusGroup) {
        continue;
      }

      // Convert timestamps to relative in place
      for (let i = 0; i < statusGroup.timestamps.length; i++) {
        statusGroup.timestamps[i] =
          Math.floor(statusGroup.timestamps[i] / 1000) - startTime;
      }

      // Map to array of objects including crash data if present
      const runs = statusGroup.timestamps.map((ts, i) => {
        const run = {
          timestamp: ts,
          taskIdId: statusGroup.taskIdIds[i],
          duration: statusGroup.durations[i],
        };
        // Include crash data if this is a CRASH status group
        if (statusGroup.crashSignatureIds) {
          run.crashSignatureId = statusGroup.crashSignatureIds[i];
        }
        if (statusGroup.minidumps) {
          run.minidump = statusGroup.minidumps[i];
        }
        // Include message data if this status group has messages
        if (statusGroup.messageIds) {
          run.messageId = statusGroup.messageIds[i];
        }
        return run;
      });

      // Sort by timestamp
      runs.sort((a, b) => a.timestamp - b.timestamp);

      // Apply differential compression in place for timestamps
      let previousTimestamp = 0;
      for (const run of runs) {
        const currentTimestamp = run.timestamp;
        run.timestamp = currentTimestamp - previousTimestamp;
        previousTimestamp = currentTimestamp;
      }

      // Update in place
      statusGroup.taskIdIds = runs.map(run => run.taskIdId);
      statusGroup.durations = runs.map(run => run.duration);
      statusGroup.timestamps = runs.map(run => run.timestamp);
      // Update crash data arrays if present
      if (statusGroup.crashSignatureIds) {
        statusGroup.crashSignatureIds = runs.map(run => run.crashSignatureId);
      }
      if (statusGroup.minidumps) {
        statusGroup.minidumps = runs.map(run => run.minidump);
      }
      // Update message data arrays if present
      if (statusGroup.messageIds) {
        statusGroup.messageIds = runs.map(run => run.messageId);
      }
    }
  }

  // Finalize the error/warning marker data folded in during job processing.
  // Occurrences carry no timestamp: all markers from a job share the job's time
  // (recoverable from the task ID via the resources file), and the only
  // aggregation is per-day, determined by which daily file a marker is in.
  let markerData = null;
  if (markerAccumulator) {
    try {
      const markerStructure = sortMarkerStringTables(
        markerAccumulator.finalize()
      );
      markerData = {
        metadata: {
          ...metadata,
          startTime,
          generatedAt: new Date().toISOString(),
          jobCount: jobs.length,
          processedJobCount: jobResults.length,
          invalidJobCount,
          markerCounts: markerOccurrencesByName(markerStructure),
        },
        tables: markerStructure.tables,
        messages: markerStructure.messages,
        taskInfo: markerStructure.taskInfo,
        testInfo: markerStructure.testInfo,
        markers: markerStructure.markers,
      };
    } catch (error) {
      console.error("Error building error/warning marker data:", error);
    }
  }

  // Build output with metadata
  return {
    testData: {
      metadata: {
        ...metadata,
        startTime,
        generatedAt: new Date().toISOString(),
        jobCount: jobs.length,
        processedJobCount: jobResults.length,
        invalidJobCount,
      },
      tables: dataStructure.tables,
      taskInfo: dataStructure.taskInfo,
      testInfo: dataStructure.testInfo,
      testRuns: dataStructure.testRuns,
    },
    resourceData: createResourceUsageData(jobResults),
    markerData,
  };
}

async function processRevisionData(project, revision, forceRefetch = false) {
  console.log(`Fetching ${HARNESS} test data for ${project}:${revision}`);
  console.log(`=== Processing ${project}:${revision} ===`);

  const cacheFile = path.join(
    OUTPUT_DIR,
    `${HARNESS}-${project}-${revision}.json`
  );

  // Check if we already have data for this revision
  if (fs.existsSync(cacheFile) && !forceRefetch) {
    console.log(`Data for ${project}:${revision} already exists. Skipping.`);
    return null;
  }

  if (forceRefetch) {
    console.log(
      `Force flag detected, re-fetching data for ${project}:${revision}...`
    );
  }

  try {
    // Fetch push ID from revision
    const pushId = await fetchCommitData(project, revision);

    // Fetch jobs for the push
    const jobs = await fetchPushJobs(project, pushId);

    if (jobs.length === 0) {
      console.log(`No ${HARNESS} jobs found for ${project}:${revision}.`);
      return null;
    }

    // Use the last_modified time of the first job as start time
    const startTime = jobs.length
      ? Math.floor(new Date(jobs[0].start_time).getTime() / 1000)
      : Math.floor(Date.now() / 1000);

    const output = await processJobsAndCreateData(
      jobs,
      `${project}-${revision}`,
      startTime,
      {
        project,
        revision,
        pushId,
      }
    );

    if (!output) {
      return null;
    }

    saveJsonFile(output.testData, cacheFile);
    const resourceCacheFile = path.join(
      OUTPUT_DIR,
      `${HARNESS}-${project}-${revision}-resources.json`
    );
    saveJsonFile(output.resourceData, resourceCacheFile);
    const errorsCacheFile = path.join(
      OUTPUT_DIR,
      `${HARNESS}-${project}-${revision}-errors.json`
    );
    saveMarkerData(output.markerData, errorsCacheFile);

    return output;
  } catch (error) {
    console.error(`Error processing ${project}:${revision}:`, error);
    return null;
  }
}

// Fetch previous run metadata from Taskcluster
async function fetchPreviousRunData() {
  try {
    // Fetch task info for the current task to get the index name from the routes.
    const taskUrl = `${TASKCLUSTER_BASE_URL}/api/queue/v1/task/${process.env.TASK_ID}`;
    const taskData = await fetchJson(taskUrl);
    if (!taskData) {
      console.log(`Failed to fetch task info from ${taskUrl}`);
      return;
    }

    const routes = taskData.routes || [];
    // Find a route that starts with "index." and contains ".latest."
    const latestRoute = routes.find(
      route => route.startsWith("index.") && route.includes(".latest.")
    );
    if (!latestRoute) {
      console.log(
        `No route found with 'index.' prefix and '.latest.' in name. Available routes: ${JSON.stringify(routes)}`
      );
      return;
    }

    // Remove "index." prefix from route to get index name
    const indexName = latestRoute.replace(/^index\./, "");
    console.log(`Using index: ${indexName}`);

    // Store artifacts URL for later use by processDateData
    const artifactsUrl = `${TASKCLUSTER_BASE_URL}/api/index/v1/task/${indexName}/artifacts/public`;

    // Fetch the index.json from the previous run
    const indexUrl = `${artifactsUrl}/index.json`;
    console.log(`Fetching previous run data from ${indexUrl}`);
    const indexData = await fetchJson(indexUrl);
    if (!indexData) {
      console.log(`Failed to fetch index.json from ${indexUrl}`);
      return;
    }

    const dates = indexData.dates || [];

    console.log(`Found ${dates.length} dates in previous run`);

    previousRunData = {
      dates: new Set(dates),
      artifactsUrl,
    };

    // Fetch previous stats and populate dailyStatsMap
    const statsUrl = `${artifactsUrl}/${HARNESS}-stats.json`;
    console.log(`Fetching previous stats from ${statsUrl}...`);
    const previousStats = await fetchJson(statsUrl);
    if (previousStats && previousStats.dates) {
      console.log(`Found ${previousStats.dates.length} days of previous stats`);
      for (let i = 0; i < previousStats.dates.length; i++) {
        const date = previousStats.dates[i];
        const entry = {
          totalTestRuns: previousStats.totalTestRuns[i],
          failedTestRuns: previousStats.failedTestRuns[i],
          skippedTestRuns: previousStats.skippedTestRuns[i],
          processedJobCount: previousStats.processedJobCount[i],
          failedJobs: previousStats.failedJobs[i],
          invalidJobs: previousStats.invalidJobs[i],
          ignoredJobs: previousStats.ignoredJobs[i],
        };
        if (previousStats.flavors) {
          entry.flavors = {};
          for (const [flavor, data] of Object.entries(previousStats.flavors)) {
            entry.flavors[flavor] = {
              totalTestRuns: data.totalTestRuns[i],
              failedTestRuns: data.failedTestRuns[i],
              skippedTestRuns: data.skippedTestRuns[i],
              processedJobCount: data.processedJobCount[i],
              failedJobs: data.failedJobs[i],
              ignoredJobs: data.ignoredJobs[i],
            };
          }
        }
        if (previousStats.markerCounts) {
          entry.markerCounts = {};
          for (const [name, counts] of Object.entries(
            previousStats.markerCounts
          )) {
            entry.markerCounts[name] = counts[i];
          }
        }
        dailyStatsMap.set(date, entry);
      }
    }

    console.log("Previous run metadata loaded\n");
  } catch (error) {
    console.log(`Error fetching previous run metadata: ${error.message}`);
  }
}

// Process data for a single date
async function processDateData(
  targetDate,
  forceRefetch = false,
  acceptIncomplete = false
) {
  const timingsFilename = `${HARNESS}-${targetDate}.json`;
  const resourcesFilename = `${HARNESS}-${targetDate}-resources.json`;
  const errorsFilename = `${HARNESS}-${targetDate}-errors.json`;
  const timingsPath = path.join(OUTPUT_DIR, timingsFilename);
  const resourcesPath = path.join(OUTPUT_DIR, resourcesFilename);
  const errorsPath = path.join(OUTPUT_DIR, errorsFilename);

  // Check if we already have data for this date
  if (fs.existsSync(timingsPath) && !forceRefetch) {
    console.log(`Data for ${targetDate} already exists, recomputing stats.`);
    const testData = JSON.parse(fs.readFileSync(timingsPath, "utf-8"));
    const existing = dailyStatsMap.get(targetDate);
    calculateStatsFromData(
      testData,
      targetDate,
      existing?.ignoredJobs,
      existing?.failedJobs
    );
    return;
  }

  // Fetch jobs list first (needed for verification)
  let allDateJobs;
  try {
    allDateJobs = await fetchHarnessData(targetDate);
    if (allDateJobs.length === 0) {
      console.log(`No jobs found for ${targetDate}.`);
      return;
    }
  } catch (error) {
    console.error(`Error fetching jobs for ${targetDate}:`, error);
    return;
  }

  // Filter out ignored jobs
  const jobs = allDateJobs.filter(job => !ignoreTasksCache.has(job.task));
  const ignoredJobsCount = allDateJobs.length - jobs.length;
  const failedJobsCount = jobs.filter(j => j.state === "failed").length;

  // Per-flavor job counts from the raw job list
  let flavorJobCounts = null;
  if (HARNESS === "mochitest") {
    flavorJobCounts = {};
    for (const job of allDateJobs) {
      const flavor = classifyMochitestFlavor(job.name);
      if (flavor === "other") {
        continue;
      }
      if (!flavorJobCounts[flavor]) {
        flavorJobCounts[flavor] = { total: 0, failed: 0, ignored: 0 };
      }
      if (ignoreTasksCache.has(job.task)) {
        flavorJobCounts[flavor].ignored++;
      } else {
        flavorJobCounts[flavor].total++;
        if (job.state === "failed") {
          flavorJobCounts[flavor].failed++;
        }
      }
    }
  }

  console.log(
    `Found ${allDateJobs.length} jobs for ${targetDate} (${ignoredJobsCount} ignored, ${jobs.length} to process)`
  );

  if (jobs.length === 0) {
    console.log(`No jobs to process for ${targetDate} after filtering.`);
    return;
  }

  // Try to fetch from previous run if available and not forcing refetch
  if (
    !forceRefetch &&
    previousRunData &&
    previousRunData.dates.has(targetDate)
  ) {
    try {
      const [timings, resources, errors] = await Promise.all([
        fetchJson(`${previousRunData.artifactsUrl}/${timingsFilename}`),
        fetchJson(`${previousRunData.artifactsUrl}/${resourcesFilename}`),
        fetchJson(`${previousRunData.artifactsUrl}/${errorsFilename}`, true),
      ]);

      if (timings && resources) {
        const expectedJobCount = jobs.length;
        const actualProcessedCount =
          timings.metadata.processedJobCount +
          (timings.metadata.invalidJobCount || 0);

        // Check if previous run processed fewer jobs (had retryable errors or incomplete data)
        if (!acceptIncomplete && actualProcessedCount < expectedJobCount) {
          const missingJobs = expectedJobCount - actualProcessedCount;
          console.log(
            `Ignoring artifact from previous run: missing ${missingJobs} jobs (expected ${expectedJobCount}, got ${actualProcessedCount})`
          );
        } else {
          console.log(`Fetched valid artifact from previous run.`);
          saveJsonFile(timings, timingsPath);
          saveJsonFile(resources, resourcesPath);
          // Missing for the days generated before we started producing it.
          const savedErrors = saveMarkerData(errors, errorsPath);

          calculateStatsFromData(
            timings,
            targetDate,
            ignoredJobsCount,
            failedJobsCount,
            flavorJobCounts
          );
          if (savedErrors) {
            recordMarkerCounts(targetDate, errors);
          }
          return;
        }
      } else {
        console.log(
          `Error fetching artifact from previous run: artifact not found`
        );
      }
    } catch (error) {
      console.log(
        `Error fetching artifact from previous run: ${error.message}`
      );
    }
  }

  if (acceptIncomplete) {
    console.log(`No previous data available for ${targetDate}, skipping.`);
    return;
  }

  if (forceRefetch) {
    console.log(`Force flag detected, re-fetching data for ${targetDate}...`);
  }

  try {
    // Calculate start of day timestamp for relative time calculation
    const startOfDay = new Date(targetDate + "T00:00:00.000Z");
    const startTime = Math.floor(startOfDay.getTime() / 1000); // Convert to seconds

    const output = await processJobsAndCreateData(jobs, targetDate, startTime, {
      date: targetDate,
    });
    if (!output) {
      return;
    }

    saveJsonFile(output.testData, timingsPath);
    saveJsonFile(output.resourceData, resourcesPath);

    calculateStatsFromData(
      output.testData,
      targetDate,
      ignoredJobsCount,
      failedJobsCount,
      flavorJobCounts
    );

    // Only report the counts if we have a file the dashboard can drill into.
    if (saveMarkerData(output.markerData, errorsPath)) {
      recordMarkerCounts(targetDate, output.markerData);
    }
  } catch (error) {
    console.error(`Error processing ${targetDate}:`, error);
  }
}

// eslint-disable-next-line complexity
async function createAggregatedFailuresFile(dates) {
  console.log(
    `\n=== Creating aggregated failures file from ${dates.length} days ===`
  );

  const dailyFiles = [];
  for (const date of dates) {
    const filePath = path.join(OUTPUT_DIR, `${HARNESS}-${date}.json`);
    if (fs.existsSync(filePath)) {
      dailyFiles.push({ date, filePath });
    }
  }

  if (dailyFiles.length === 0) {
    console.log("No daily files found to aggregate");
    return;
  }

  console.log(`Found ${dailyFiles.length} daily files to aggregate`);

  const startDate = dates[dates.length - 1];
  const endDate = dates[0];
  const startTime = Math.floor(
    new Date(startDate + "T00:00:00.000Z").getTime() / 1000
  );

  const mergedTables = {
    jobNames: [],
    testPaths: [],
    testNames: [],
    repositories: [],
    statuses: [],
    taskIds: [],
    messages: [],
    crashSignatures: [],
    components: [],
    commitIds: [],
  };

  const stringMaps = {
    jobNames: new Map(),
    testPaths: new Map(),
    testNames: new Map(),
    repositories: new Map(),
    statuses: new Map(),
    taskIds: new Map(),
    messages: new Map(),
    crashSignatures: new Map(),
    components: new Map(),
    commitIds: new Map(),
  };

  function addToMergedTable(tableName, value) {
    if (value === null || value === undefined) {
      return null;
    }
    const map = stringMaps[tableName];
    let index = map.get(value);
    if (index === undefined) {
      index = mergedTables[tableName].length;
      mergedTables[tableName].push(value);
      map.set(value, index);
    }
    return index;
  }

  const mergedTaskInfo = {
    repositoryIds: [],
    jobNameIds: [],
    commitIds: [],
  };

  const mergedTestInfo = {
    testPathIds: [],
    testNameIds: [],
    componentIds: [],
  };

  const testPathMap = new Map();
  const mergedTestRuns = [];

  for (let fileIdx = 0; fileIdx < dailyFiles.length; fileIdx++) {
    const { date, filePath } = dailyFiles[fileIdx];
    console.log(`Processing ${fileIdx + 1}/${dailyFiles.length}: ${date}...`);

    const data = JSON.parse(fs.readFileSync(filePath, "utf-8"));

    const dayStartTime = data.metadata.startTime;
    const timeOffset = dayStartTime - startTime;

    for (let testId = 0; testId < data.testRuns.length; testId++) {
      const testGroup = data.testRuns[testId];
      if (!testGroup) {
        continue;
      }

      const testPathId = data.testInfo.testPathIds[testId];
      const testNameId = data.testInfo.testNameIds[testId];
      const componentId = data.testInfo.componentIds[testId];

      const testPath = data.tables.testPaths[testPathId];
      const testName = data.tables.testNames[testNameId];
      const fullPath = testPath ? `${testPath}/${testName}` : testName;

      let mergedTestId = testPathMap.get(fullPath);
      if (mergedTestId === undefined) {
        mergedTestId = mergedTestInfo.testPathIds.length;

        const mergedTestPathId = addToMergedTable("testPaths", testPath);
        const mergedTestNameId = addToMergedTable("testNames", testName);
        const component =
          componentId !== null ? data.tables.components[componentId] : null;
        const mergedComponentId = addToMergedTable("components", component);

        mergedTestInfo.testPathIds.push(mergedTestPathId);
        mergedTestInfo.testNameIds.push(mergedTestNameId);
        mergedTestInfo.componentIds.push(mergedComponentId);

        testPathMap.set(fullPath, mergedTestId);
        mergedTestRuns[mergedTestId] = [];
      }

      for (let statusId = 0; statusId < testGroup.length; statusId++) {
        const statusGroup = testGroup[statusId];
        if (!statusGroup) {
          continue;
        }

        const status = data.tables.statuses[statusId];
        const mergedStatusId = addToMergedTable("statuses", status);
        const isPass = status.startsWith("PASS");
        const isCrash = status === "CRASH";

        let group = mergedTestRuns[mergedTestId][mergedStatusId];
        if (!group) {
          group = {
            repositoryIds: [],
            jobNameIds: [],
            timestamps: [],
            durations: [],
          };

          if (!isPass) {
            group.taskIdIds = [];
            if (statusGroup.messageIds) {
              group.messageIds = [];
            }
          }

          if (isCrash) {
            group.crashSignatureIds = [];
            group.minidumps = [];
          }

          mergedTestRuns[mergedTestId][mergedStatusId] = group;
        }

        let absoluteTimestamp = 0;
        for (let i = 0; i < statusGroup.taskIdIds.length; i++) {
          absoluteTimestamp += statusGroup.timestamps[i];

          // Skip platform-irrelevant tests (SKIP with run-if messages)
          if (
            status === "SKIP" &&
            data.tables.messages[statusGroup.messageIds?.[i]]?.startsWith(
              "run-if"
            )
          ) {
            continue;
          }

          const taskIdId = statusGroup.taskIdIds[i];
          const taskIdString = data.tables.taskIds[taskIdId];
          const repositoryId = data.taskInfo.repositoryIds[taskIdId];
          const jobNameId = data.taskInfo.jobNameIds[taskIdId];
          const commitId = data.taskInfo.commitIds[taskIdId];

          const repository = data.tables.repositories[repositoryId];
          const jobName = data.tables.jobNames[jobNameId];
          const commitIdString =
            commitId !== null ? data.tables.commitIds[commitId] : null;

          const mergedRepositoryId = addToMergedTable(
            "repositories",
            repository
          );
          const mergedJobNameId = addToMergedTable("jobNames", jobName);
          const mergedCommitId = addToMergedTable("commitIds", commitIdString);

          group.repositoryIds.push(mergedRepositoryId);
          group.jobNameIds.push(mergedJobNameId);
          group.timestamps.push(absoluteTimestamp + timeOffset);
          group.durations.push(statusGroup.durations[i]);

          if (isPass) {
            continue;
          }

          const mergedTaskIdId = addToMergedTable("taskIds", taskIdString);

          if (mergedTaskInfo.repositoryIds[mergedTaskIdId] === undefined) {
            mergedTaskInfo.repositoryIds[mergedTaskIdId] = mergedRepositoryId;
            mergedTaskInfo.jobNameIds[mergedTaskIdId] = mergedJobNameId;
            mergedTaskInfo.commitIds[mergedTaskIdId] = mergedCommitId;
          }

          group.taskIdIds.push(mergedTaskIdId);

          if (group.messageIds) {
            const messageId = statusGroup.messageIds?.[i];
            if (typeof messageId === "number") {
              const message = data.tables.messages[messageId];
              group.messageIds.push(addToMergedTable("messages", message));
            } else {
              group.messageIds.push(null);
            }
          } else if (statusGroup.messageIds) {
            console.warn(
              `Losing messageIds data for test ${testPath}, status ${status} (not present in first day)`
            );
          }

          if (isCrash) {
            const crashSigId = statusGroup.crashSignatureIds?.[i];
            if (typeof crashSigId === "number") {
              const crashSig = data.tables.crashSignatures[crashSigId];
              group.crashSignatureIds.push(
                addToMergedTable("crashSignatures", crashSig)
              );
            } else {
              group.crashSignatureIds.push(null);
            }

            group.minidumps.push(statusGroup.minidumps?.[i] ?? null);
          }
        }
      }
    }
  }

  function compareNullable(a, b) {
    if (a === b) {
      return 0;
    }
    if (a === null || a === undefined) {
      return 1;
    }
    if (b === null || b === undefined) {
      return -1;
    }
    return a - b;
  }

  function aggregateRunsByDay(
    statusGroup,
    {
      includeMessages = false,
      includeTaskIds = false,
      includeJobNames = false,
      includeDurations = false,
    } = {}
  ) {
    const buckets = new Map();
    const length = statusGroup.timestamps.length;

    function getOrCreateBucket(
      key,
      dayBucket,
      messageId,
      crashSignatureId,
      jobNameId
    ) {
      let bucket = buckets.get(key);
      if (!bucket) {
        bucket = { day: dayBucket, count: 0, messageId, crashSignatureId };
        if (includeTaskIds) {
          bucket.taskIdIds = [];
          bucket.minidumps = [];
        }
        if (includeDurations) {
          bucket.durations = [];
        }
        if (includeJobNames) {
          bucket.jobNameId = jobNameId;
        }
        buckets.set(key, bucket);
      }
      return bucket;
    }

    for (let i = 0; i < length; i++) {
      const dayBucket = Math.floor(statusGroup.timestamps[i] / 86400);
      let key = `${dayBucket}`;

      const messageId = statusGroup.messageIds?.[i];
      const crashSignatureId = statusGroup.crashSignatureIds?.[i];
      const jobNameId = statusGroup.jobNameIds?.[i];

      if (includeJobNames && jobNameId !== undefined) {
        key += `:j${jobNameId}`;
      }

      if (includeMessages && typeof messageId === "number") {
        key += `:m${messageId}`;
      } else if (includeMessages && typeof crashSignatureId === "number") {
        key += `:c${crashSignatureId}`;
      }

      const bucket = getOrCreateBucket(
        key,
        dayBucket,
        messageId,
        crashSignatureId,
        jobNameId
      );
      bucket.count++;
      if (includeTaskIds && statusGroup.taskIdIds) {
        bucket.taskIdIds.push(statusGroup.taskIdIds[i]);
      }
      if (includeTaskIds && statusGroup.minidumps) {
        bucket.minidumps.push(statusGroup.minidumps[i] ?? null);
      }
      if (includeDurations && statusGroup.durations) {
        bucket.durations.push(statusGroup.durations[i]);
      }
    }

    const aggregated = Array.from(buckets.values()).sort((a, b) => {
      return (
        a.day - b.day ||
        compareNullable(a.jobNameId, b.jobNameId) ||
        compareNullable(a.messageId, b.messageId) ||
        compareNullable(a.crashSignatureId, b.crashSignatureId)
      );
    });

    const days = [];
    let previousBucket = 0;
    for (const item of aggregated) {
      days.push(item.day - previousBucket);
      previousBucket = item.day;
    }

    const result = {
      days,
    };

    if (includeTaskIds) {
      result.taskIdIds = aggregated.map(a => a.taskIdIds);
    } else if (includeDurations) {
      result.durations = aggregated.map(a => a.durations);
    } else {
      result.counts = aggregated.map(a => a.count);
    }

    if (includeJobNames) {
      result.jobNameIds = aggregated.map(a => a.jobNameId ?? null);
    }

    if (includeMessages) {
      if (aggregated.some(a => "messageId" in a && a.messageId !== undefined)) {
        result.messageIds = aggregated.map(a => a.messageId ?? null);
      }
      if (
        aggregated.some(
          a => "crashSignatureId" in a && a.crashSignatureId !== undefined
        )
      ) {
        result.crashSignatureIds = aggregated.map(
          a => a.crashSignatureId ?? null
        );
      }
      if (includeTaskIds && aggregated.some(a => a.minidumps?.length)) {
        result.minidumps = aggregated.map(a => a.minidumps);
      }
    }

    return result;
  }

  console.log("Aggregating passing test runs by day...");

  const finalTestRuns = [];

  for (let testId = 0; testId < mergedTestRuns.length; testId++) {
    const testGroup = mergedTestRuns[testId];
    if (!testGroup) {
      continue;
    }

    finalTestRuns[testId] = [];

    for (let statusId = 0; statusId < testGroup.length; statusId++) {
      const statusGroup = testGroup[statusId];
      if (!statusGroup?.timestamps?.length) {
        continue;
      }

      const status = mergedTables.statuses[statusId];
      const isPass = status.startsWith("PASS");

      if (isPass) {
        finalTestRuns[testId][statusId] = aggregateRunsByDay(statusGroup);
      } else {
        finalTestRuns[testId][statusId] = aggregateRunsByDay(statusGroup, {
          includeMessages: true,
          includeTaskIds: true,
        });
      }
    }
  }

  const testsWithFailures = finalTestRuns.filter(testGroup =>
    testGroup?.some(
      (sg, idx) => sg && !mergedTables.statuses[idx].startsWith("PASS")
    )
  ).length;

  console.log("Sorting string tables by frequency...");

  // Sort string tables by frequency for better compression
  const dataStructure = {
    tables: mergedTables,
    taskInfo: mergedTaskInfo,
    testInfo: mergedTestInfo,
    testRuns: finalTestRuns,
  };

  const sortedData = sortStringTablesByFrequency(dataStructure);

  const outputData = {
    metadata: {
      startDate,
      endDate,
      days: dates.length,
      startTime,
      generatedAt: new Date().toISOString(),
      totalTestCount: mergedTestInfo.testPathIds.length,
      testsWithFailures,
      aggregatedFrom: dailyFiles.map(f => path.basename(f.filePath)),
    },
    tables: sortedData.tables,
    taskInfo: sortedData.taskInfo,
    testInfo: sortedData.testInfo,
    testRuns: sortedData.testRuns,
  };

  const outputFileWithDetails = path.join(
    OUTPUT_DIR,
    `${HARNESS}-issues-with-taskids.json`
  );
  saveJsonFile(outputData, outputFileWithDetails);

  // Create small file with all statuses aggregated
  console.log("Creating small aggregated version...");

  const smallTestRuns = sortedData.testRuns.map(testGroup => {
    if (!testGroup) {
      return testGroup;
    }
    return testGroup.map(statusGroup => {
      if (!statusGroup) {
        return statusGroup;
      }
      if (statusGroup.counts) {
        return statusGroup;
      }

      const result = {
        counts: statusGroup.taskIdIds.map(arr => arr.length),
        days: statusGroup.days,
      };

      if (statusGroup.messageIds) {
        result.messageIds = statusGroup.messageIds;
      }

      if (statusGroup.crashSignatureIds) {
        result.crashSignatureIds = statusGroup.crashSignatureIds;
      }

      return result;
    });
  });

  const smallOutput = {
    metadata: outputData.metadata,
    tables: {
      testPaths: sortedData.tables.testPaths,
      testNames: sortedData.tables.testNames,
      statuses: sortedData.tables.statuses,
      messages: sortedData.tables.messages,
      crashSignatures: sortedData.tables.crashSignatures,
      components: sortedData.tables.components,
    },
    testInfo: sortedData.testInfo,
    testRuns: smallTestRuns,
  };

  const outputFileSmall = path.join(OUTPUT_DIR, `${HARNESS}-issues.json`);
  saveJsonFile(smallOutput, outputFileSmall);

  console.log(
    `Successfully created aggregated files with ${outputData.metadata.totalTestCount} tests`
  );
  console.log(`  Tests with failures: ${testsWithFailures}`);

  // --- Bucket file generation ---
  const TOTAL_BUCKETS = 64;

  function getBucketIndex(fullPath) {
    let hash = 0;
    for (let i = 0; i < fullPath.length; i++) {
      hash = ((hash << 5) - hash + fullPath.charCodeAt(i)) | 0;
    }
    return ((hash % TOTAL_BUCKETS) + TOTAL_BUCKETS) % TOTAL_BUCKETS;
  }

  console.log("\nGenerating bucket files...");

  // Build jobNameBaseMap: merged jobNameId -> { baseId, chunk }
  // Strip chunk suffixes like "-1", "-2" from job names.
  const bucketJobNames = [];
  const bucketJobNameMap = new Map();
  const jobNameBaseMap = new Map();

  for (let id = 0; id < mergedTables.jobNames.length; id++) {
    const jobName = mergedTables.jobNames[id];
    let baseName = jobName;
    let chunkNumber = null;
    const chunkMatch = jobName.match(/^(.+)-(\d+)(-cf)?$/);
    if (chunkMatch) {
      baseName = chunkMatch[1] + (chunkMatch[3] || "");
      chunkNumber = parseInt(chunkMatch[2], 10);
    }

    let baseId = bucketJobNameMap.get(baseName);
    if (baseId === undefined) {
      baseId = bucketJobNames.length;
      bucketJobNames.push(baseName);
      bucketJobNameMap.set(baseName, baseId);
    }

    jobNameBaseMap.set(id, { baseId, chunk: chunkNumber });
  }

  // Build bucketTaskInfo: extend mergedTaskInfo with chunks, using base jobNameIds
  const bucketTaskInfo = {
    repositoryIds: mergedTaskInfo.repositoryIds.slice(),
    jobNameIds: mergedTaskInfo.jobNameIds.map(id => {
      if (id === undefined) {
        return undefined;
      }
      return jobNameBaseMap.get(id).baseId;
    }),
    commitIds: mergedTaskInfo.commitIds.slice(),
    chunks: mergedTaskInfo.jobNameIds.map(id => {
      if (id === undefined) {
        return null;
      }
      return jobNameBaseMap.get(id).chunk;
    }),
  };

  function aggregateTestForBucket(testId) {
    const testGroup = mergedTestRuns[testId];
    if (!testGroup) {
      return [];
    }

    const result = [];
    for (let statusId = 0; statusId < testGroup.length; statusId++) {
      const statusGroup = testGroup[statusId];
      if (!statusGroup?.timestamps?.length) {
        continue;
      }

      const status = mergedTables.statuses[statusId];
      const isPass = status.startsWith("PASS");
      const isSkip = status === "SKIP";

      if (isPass) {
        const sg = {
          timestamps: statusGroup.timestamps,
          durations: statusGroup.durations,
          jobNameIds: statusGroup.jobNameIds.map(
            id => jobNameBaseMap.get(id).baseId
          ),
        };
        result[statusId] = aggregateRunsByDay(sg, {
          includeJobNames: true,
          includeDurations: true,
        });
      } else if (isSkip) {
        const sg = {
          timestamps: statusGroup.timestamps,
          jobNameIds: statusGroup.jobNameIds.map(
            id => jobNameBaseMap.get(id).baseId
          ),
          messageIds: statusGroup.messageIds,
        };
        result[statusId] = aggregateRunsByDay(sg, {
          includeMessages: true,
          includeJobNames: true,
        });
      } else {
        result[statusId] = aggregateRunsByDay(statusGroup, {
          includeMessages: true,
          includeTaskIds: true,
        });
      }
    }
    return result;
  }

  // Group tests by bucket index
  const bucketGroups = new Array(TOTAL_BUCKETS).fill(null).map(() => []);

  for (const [fullPath, testId] of testPathMap) {
    const bucketIdx = getBucketIndex(fullPath);
    bucketGroups[bucketIdx].push({ fullPath, testId });
  }

  // Write bucket files, aggregating each test on demand per bucket
  let totalBucketSize = 0;
  let nonEmptyBuckets = 0;

  for (let bucketIdx = 0; bucketIdx < TOTAL_BUCKETS; bucketIdx++) {
    const tests = bucketGroups[bucketIdx];

    // Build testInfo and testRuns for this bucket using global indices;
    // sortStringTablesByFrequency will compact out unused table entries.
    const localTestInfo = {
      testPathIds: [],
      testNameIds: [],
      componentIds: [],
    };
    const localTestRuns = [];
    let testsWithFailures = 0;

    for (let localTestId = 0; localTestId < tests.length; localTestId++) {
      const { testId } = tests[localTestId];

      localTestInfo.testPathIds.push(mergedTestInfo.testPathIds[testId]);
      localTestInfo.testNameIds.push(mergedTestInfo.testNameIds[testId]);
      localTestInfo.componentIds.push(mergedTestInfo.componentIds[testId]);

      const aggregated = aggregateTestForBucket(testId);
      localTestRuns[localTestId] = aggregated;

      if (
        aggregated.some(
          (sg, idx) => sg && !mergedTables.statuses[idx].startsWith("PASS")
        )
      ) {
        testsWithFailures++;
      }
    }

    const bucketHex = bucketIdx.toString(16).padStart(2, "0");
    const bucketFile = path.join(OUTPUT_DIR, `${HARNESS}-${bucketHex}.json`);

    const bucketData = {
      metadata: {
        startDate,
        endDate,
        days: dates.length,
        startTime,
        generatedAt: new Date().toISOString(),
        totalTestCount: tests.length,
        testsWithFailures,
        totalBuckets: TOTAL_BUCKETS,
        bucketIndex: bucketIdx,
        aggregatedFrom: dailyFiles.map(f => path.basename(f.filePath)),
      },
      tables: {
        jobNames: bucketJobNames,
        testPaths: mergedTables.testPaths,
        testNames: mergedTables.testNames,
        repositories: mergedTables.repositories,
        statuses: mergedTables.statuses,
        taskIds: mergedTables.taskIds,
        messages: mergedTables.messages,
        crashSignatures: mergedTables.crashSignatures,
        components: mergedTables.components,
        commitIds: mergedTables.commitIds,
      },
      taskInfo: bucketTaskInfo,
      testInfo: localTestInfo,
      testRuns: localTestRuns,
    };

    const sortedBucketData = sortStringTablesByFrequency(bucketData);

    saveJsonFile(
      {
        metadata: bucketData.metadata,
        tables: sortedBucketData.tables,
        taskInfo: sortedBucketData.taskInfo,
        testInfo: sortedBucketData.testInfo,
        testRuns: sortedBucketData.testRuns,
      },
      bucketFile
    );

    if (tests.length) {
      nonEmptyBuckets++;
    }
    const fileSize = fs.statSync(bucketFile).size;
    totalBucketSize += fileSize;
  }

  const totalBucketSizeMB = Math.round(totalBucketSize / (1024 * 1024));
  console.log(
    `Generated ${TOTAL_BUCKETS} bucket files (${nonEmptyBuckets} non-empty, ${totalBucketSizeMB}MB total)`
  );
}

function calculateStatsFromData(
  testData,
  targetDate,
  ignoredJobsCount = 0,
  failedJobsCount = 0,
  flavorJobCounts = null
) {
  const stats = {
    totalTestRuns: 0,
    failedTestRuns: 0,
    skippedTestRuns: 0,
    processedJobCount: testData.metadata.processedJobCount || 0,
    failedJobs: failedJobsCount,
    invalidJobs: testData.metadata.invalidJobCount || 0,
    ignoredJobs: ignoredJobsCount,
    // The marker counts don't depend on the timing data the rest of the stats is
    // computed from, so keep the ones a previous run or computation found.
    markerCounts: dailyStatsMap.get(targetDate)?.markerCounts,
  };

  const trackFlavors = HARNESS === "mochitest";
  let flavorByJobNameId, flavorStatsMap;

  if (trackFlavors) {
    flavorByJobNameId = testData.tables.jobNames.map(classifyMochitestFlavor);
    flavorStatsMap = new Map();
  }

  function addToFlavors(taskIdIds, isFailed, isSkipped) {
    if (!trackFlavors) {
      return;
    }
    for (const taskIdId of taskIdIds) {
      const jobNameId = testData.taskInfo.jobNameIds[taskIdId];
      const flavor = flavorByJobNameId[jobNameId];
      if (flavor === "other") {
        continue;
      }
      let fStats = flavorStatsMap.get(flavor);
      if (!fStats) {
        fStats = { totalTestRuns: 0, failedTestRuns: 0, skippedTestRuns: 0 };
        flavorStatsMap.set(flavor, fStats);
      }
      fStats.totalTestRuns++;
      if (isFailed) {
        fStats.failedTestRuns++;
      }
      if (isSkipped) {
        fStats.skippedTestRuns++;
      }
    }
  }

  for (const testGroup of testData.testRuns) {
    for (let statusId = 0; statusId < testGroup.length; statusId++) {
      const statusGroup = testGroup[statusId];
      if (!statusGroup) {
        continue;
      }

      const status = testData.tables.statuses[statusId];
      const runCount = statusGroup.taskIdIds.length;
      stats.totalTestRuns += runCount;

      const isFailed =
        status.startsWith("FAIL") || status === "CRASH" || status === "TIMEOUT";

      if (isFailed) {
        stats.failedTestRuns += runCount;
        addToFlavors(statusGroup.taskIdIds, true, false);
      } else if (status === "SKIP") {
        if (statusGroup.messageIds) {
          for (let i = 0; i < statusGroup.messageIds.length; i++) {
            const messageId = statusGroup.messageIds[i];
            const isRunIf =
              messageId != null &&
              testData.tables.messages[messageId].startsWith("run-if");
            if (!isRunIf) {
              stats.skippedTestRuns++;
            }
            addToFlavors([statusGroup.taskIdIds[i]], false, !isRunIf);
          }
        } else {
          stats.skippedTestRuns += runCount;
          addToFlavors(statusGroup.taskIdIds, false, true);
        }
      } else {
        addToFlavors(statusGroup.taskIdIds, false, false);
      }
    }
  }

  if (trackFlavors) {
    const flavors = {};
    for (const [flavor, fStats] of flavorStatsMap) {
      flavors[flavor] = { ...fStats };
      if (flavorJobCounts && flavorJobCounts[flavor]) {
        const jc = flavorJobCounts[flavor];
        flavors[flavor].processedJobCount = jc.total;
        flavors[flavor].failedJobs = jc.failed;
        flavors[flavor].ignoredJobs = jc.ignored;
      }
    }
    stats.flavors = flavors;
  }

  console.log(
    `  Stats: ${stats.totalTestRuns} runs, ${stats.failedTestRuns} failed, ${stats.failedJobs} failed jobs, ${stats.invalidJobs} invalid jobs, ${stats.ignoredJobs} ignored jobs`
  );

  dailyStatsMap.set(targetDate, stats);

  return stats;
}

// Record in a day's stats how many occurrences of each error and warning marker
// its errors file holds, so that the dashboard can show which days are noisy, and
// for which markers, without downloading the errors files.
function recordMarkerCounts(targetDate, markerData) {
  dailyStatsMap.get(targetDate).markerCounts =
    markerData?.metadata.markerCounts;
}

async function saveStatsFile() {
  console.log(`\n=== Generating statistics summary file ===`);

  const allDates = Array.from(dailyStatsMap.keys()).sort();
  if (allDates.length === 0) {
    console.log("No daily stats to save");
    return;
  }

  const output = {
    metadata: {
      generatedAt: new Date().toISOString(),
      harness: HARNESS,
    },
    dates: allDates,
    totalTestRuns: [],
    failedTestRuns: [],
    skippedTestRuns: [],
    processedJobCount: [],
    failedJobs: [],
    invalidJobs: [],
    ignoredJobs: [],
  };

  // Collect all flavor names across all dates
  const allFlavors = new Set();
  for (const date of allDates) {
    const stats = dailyStatsMap.get(date);
    if (stats.flavors) {
      for (const flavor of Object.keys(stats.flavors)) {
        allFlavors.add(flavor);
      }
    }
  }

  if (allFlavors.size > 0) {
    output.flavors = {};
    for (const flavor of [...allFlavors].sort()) {
      output.flavors[flavor] = {
        totalTestRuns: [],
        failedTestRuns: [],
        skippedTestRuns: [],
        processedJobCount: [],
        failedJobs: [],
        ignoredJobs: [],
      };
    }
  }

  // Same for the marker names, which only the days that have an errors file have.
  const allMarkerNames = new Set();
  for (const date of allDates) {
    const { markerCounts } = dailyStatsMap.get(date);
    if (markerCounts) {
      for (const name of Object.keys(markerCounts)) {
        allMarkerNames.add(name);
      }
    }
  }

  if (allMarkerNames.size > 0) {
    output.markerCounts = {};
    for (const name of [...allMarkerNames].sort()) {
      output.markerCounts[name] = [];
    }
  }

  for (const date of allDates) {
    const stats = dailyStatsMap.get(date);
    output.totalTestRuns.push(stats.totalTestRuns);
    output.failedTestRuns.push(stats.failedTestRuns);
    output.skippedTestRuns.push(stats.skippedTestRuns);
    output.processedJobCount.push(stats.processedJobCount);
    output.failedJobs.push(stats.failedJobs);
    output.invalidJobs.push(stats.invalidJobs);
    output.ignoredJobs.push(stats.ignoredJobs);

    // Not every date has every flavor (a flavor may not have run on a
    // given day, or flavor data may be missing for older dates carried
    // forward from a pre-flavor stats file), so fall back to 0.
    if (output.flavors) {
      for (const flavor of Object.keys(output.flavors)) {
        const fStats = stats.flavors?.[flavor];
        output.flavors[flavor].totalTestRuns.push(fStats?.totalTestRuns || 0);
        output.flavors[flavor].failedTestRuns.push(fStats?.failedTestRuns || 0);
        output.flavors[flavor].skippedTestRuns.push(
          fStats?.skippedTestRuns || 0
        );
        output.flavors[flavor].processedJobCount.push(
          fStats?.processedJobCount || 0
        );
        output.flavors[flavor].failedJobs.push(fStats?.failedJobs || 0);
        output.flavors[flavor].ignoredJobs.push(fStats?.ignoredJobs || 0);
      }
    }

    // A day that has no errors file (it predates them, or building it failed) and
    // a marker that fired on no job of the day both fall back to 0.
    if (output.markerCounts) {
      for (const name of Object.keys(output.markerCounts)) {
        output.markerCounts[name].push(stats.markerCounts?.[name] || 0);
      }
    }
  }

  const statsFileName = `${HARNESS}-stats.json`;
  saveJsonFile(output, path.join(OUTPUT_DIR, statsFileName));
  console.log(`${allDates.length} days (${allDates[0]} to ${allDates.at(-1)})`);
}

async function main() {
  const scriptStartTime = Date.now();

  // Log heap limit at startup
  const heapStats = require("v8").getHeapStatistics();
  const heapLimitMB = Math.round(heapStats.heap_size_limit / 1024 / 1024);
  console.log(`Node heap limit: ${heapLimitMB}MB`);

  const forceRefetch = process.argv.includes("--force");

  // Check for --days parameter
  let numDays = 3;
  const daysIndex = process.argv.findIndex(arg => arg === "--days");
  if (daysIndex !== -1 && daysIndex + 1 < process.argv.length) {
    const daysValue = parseInt(process.argv[daysIndex + 1]);
    if (!isNaN(daysValue) && daysValue > 0 && daysValue <= 30) {
      numDays = daysValue;
    } else {
      console.error("Error: --days must be a number between 1 and 30");
      process.exit(1);
    }
  }

  if (process.env.TASK_ID) {
    await fetchPreviousRunData();
  }

  // Fetch component mapping data
  await fetchComponentsData();

  // Check for --revision parameter (format: project:revision)
  const revisionIndex = process.argv.findIndex(arg => arg === "--revision");
  if (revisionIndex !== -1 && revisionIndex + 1 < process.argv.length) {
    const revisionArg = process.argv[revisionIndex + 1];
    const parts = revisionArg.split(":");

    if (parts.length !== 2) {
      console.error(
        "Error: --revision must be in format project:revision (e.g., try:abc123 or autoland:def456)"
      );
      process.exit(1);
    }

    const [project, revision] = parts;
    const output = await processRevisionData(project, revision, forceRefetch);

    if (output) {
      console.log("Successfully processed revision data.");
    } else {
      console.log("\nNo data was successfully processed.");
    }
    return;
  }

  // Check for --try option (shortcut for --revision try:...)
  const tryIndex = process.argv.findIndex(arg => arg === "--try");
  if (tryIndex !== -1 && tryIndex + 1 < process.argv.length) {
    const revision = process.argv[tryIndex + 1];
    const output = await processRevisionData("try", revision, forceRefetch);

    if (output) {
      console.log("Successfully processed try commit data.");
    } else {
      console.log("\nNo data was successfully processed.");
    }
    return;
  }

  // Fetch data for the specified number of days
  const dates = [];
  for (let i = 1; i <= numDays; i++) {
    dates.push(getDateString(i));
  }

  console.log(
    `Fetching ${HARNESS} test data for the last ${numDays} day${numDays > 1 ? "s" : ""}: ${dates.join(", ")}`
  );

  const TIME_LIMIT_HOURS = 1.5;
  const TIME_LIMIT_MS = TIME_LIMIT_HOURS * 60 * 60 * 1000;
  let acceptIncomplete = false;

  for (const date of dates) {
    console.log(`\n=== Processing ${date} ===`);
    await processDateData(date, forceRefetch, acceptIncomplete);

    // After the time limit, accept incomplete data from the previous run
    // instead of re-processing from scratch, to avoid losing data entirely.
    if (!acceptIncomplete) {
      const elapsedTime = Date.now() - scriptStartTime;
      if (elapsedTime > TIME_LIMIT_MS) {
        const remainingDates = dates.length - dates.indexOf(date) - 1;
        if (remainingDates > 0) {
          console.log(
            `\nStopping full processing after ${TIME_LIMIT_HOURS} hours. Accepting incomplete previous data for ${remainingDates} remaining date${remainingDates > 1 ? "s" : ""}.`
          );
        }
        acceptIncomplete = true;
      }
    }
  }

  // Clear caches to free memory before aggregation
  allJobsCache = null;
  componentsData = null;

  // Create index file with available dates
  const indexFile = path.join(OUTPUT_DIR, "index.json");
  const availableDates = [];

  // Scan for all harness-*.json files in the output directory
  const files = fs.readdirSync(OUTPUT_DIR);
  const pattern = new RegExp(`^${HARNESS}-(\\d{4}-\\d{2}-\\d{2})\\.json$`);
  files.forEach(file => {
    const match = file.match(pattern);
    if (match) {
      availableDates.push(match[1]);
    }
  });

  // Sort dates in descending order (newest first)
  availableDates.sort((a, b) => b.localeCompare(a));

  fs.writeFileSync(
    indexFile,
    JSON.stringify({ dates: availableDates }, null, 2)
  );
  console.log(
    `\nIndex file saved as ${indexFile} with ${availableDates.length} dates`
  );

  // Generate statistics summary file
  await saveStatsFile();

  // Create aggregated failures file if processing multiple days
  if (dates.length > 1) {
    await createAggregatedFailuresFile(dates);
  }
}

main().catch(console.error);