Add startLine, startColumn, sourceMapURL and rename uuid to id in source table (#5882)

The backend side of this PR:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1999953
- https://bugzilla.mozilla.org/show_bug.cgi?id=2016667

This PR improves the source table overall by:
- Adding startLine and startColumn to the source table. This is going to
be used for showing correct line numbers in the source view when the
source is coming from an inline script. And also it will be useful for
source map/prettify in the future.
- Adding sourceMapURL if the source contains this information. This is
needed for us implement the source map resolution.
- Renaming `uuid` field to `id`. This field was initially containing
uuid fields, but they were replaced to contain hashes instead. It's
better to have an implementation agnostic name for this field in
general.

Author: canova

docs-developer/CHANGELOG-formats.md

@@ -6,6 +6,14 @@ Note that this is not an exhaustive list. Processed profile format upgraders can
 
 ## Processed profile format
 
+### Version 61
+
+The `SourceTable` in `profile.shared.sources` was updated:
+
+- The `uuid` field was renamed to `id`.
+- Two new fields `startLine` and `startColumn` were added (1-based). These describe the start position of the script within its resource, useful for inline scripts. If the source covers the entire file, use `(1, 1)`.
+- A new `sourceMapURL` field was added. Use `null` for entries with no source map URL.
+
 ### Version 60
 
 The following tables have moved into `profile.shared`: `stackTable`, `frameTable`, `funcTable`, `resourceTable`, `nativeSymbols`. They are no longer per-thread.
@@ -137,6 +145,14 @@ Older versions are not documented in this changelog but can be found in [process
 
 ## Gecko profile format
 
+### Version 34
+
+The `SourceTable` schema was updated:
+
+- The `uuid` field was renamed to `id`.
+- Two new fields `startLine` and `startColumn` were added (1-based). If the source covers the entire file, use `(1, 1)`.
+- A new `sourceMapURL` field was added. It can be omitted if it is the last element in the row and there is no source map URL.
+
 ### Version 33
 
 The `sources` field in the Gecko profile format is now non-optional. An upgrader was added that creates an empty `SourceTable` for profiles that don't have one.

src/app-logic/constants.ts

@@ -7,12 +7,12 @@ import type { MarkerPhase } from 'firefox-profiler/types';
 // The current version of the Gecko profile format.
 // Please don't forget to update the gecko profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const GECKO_PROFILE_VERSION = 33;
+export const GECKO_PROFILE_VERSION = 34;
 
 // The current version of the "processed" profile format.
 // Please don't forget to update the processed profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const PROCESSED_PROFILE_VERSION = 60;
+export const PROCESSED_PROFILE_VERSION = 61;
 
 // The following are the margin sizes for the left and right of the timeline. Independent
 // components need to share these values.

src/components/app/SourceCodeFetcher.tsx

@@ -11,7 +11,7 @@ import {
   getSymbolServerUrl,
   getSourceViewFile,
   getSourceViewSourceIndex,
-  getSourceViewSourceUuid,
+  getSourceViewSourceId,
 } from 'firefox-profiler/selectors';
 import {
   beginLoadingSourceCodeFromUrl,
@@ -36,7 +36,7 @@ import type {
 type StateProps = {
   readonly sourceViewFile: string | null;
   readonly sourceViewSourceIndex: IndexIntoSourceTable | null;
-  readonly sourceViewSourceUuid: string | null;
+  readonly sourceViewSourceId: string | null;
   readonly sourceViewCode: SourceCodeStatus | void;
   readonly symbolServerUrl: string;
   readonly profile: Profile | null;
@@ -68,7 +68,7 @@ class SourceCodeFetcherImpl extends React.PureComponent<Props> {
       sourceViewSourceIndex,
       sourceViewCode,
       sourceViewFile,
-      sourceViewSourceUuid,
+      sourceViewSourceId,
       beginLoadingSourceCodeFromUrl,
       beginLoadingSourceCodeFromBrowserConnection,
       finishLoadingSourceCode,
@@ -105,7 +105,7 @@ class SourceCodeFetcherImpl extends React.PureComponent<Props> {
 
     const fetchSourceResult = await fetchSource(
       sourceViewFile,
-      sourceViewSourceUuid,
+      sourceViewSourceId,
       symbolServerUrl,
       addressProof,
       this._archiveCache,
@@ -137,7 +137,7 @@ export const SourceCodeFetcher = explicitConnect<{}, StateProps, DispatchProps>(
     mapStateToProps: (state) => ({
       sourceViewSourceIndex: getSourceViewSourceIndex(state),
       sourceViewFile: getSourceViewFile(state),
-      sourceViewSourceUuid: getSourceViewSourceUuid(state),
+      sourceViewSourceId: getSourceViewSourceId(state),
       sourceViewCode: getSourceViewCode(state),
       symbolServerUrl: getSymbolServerUrl(state),
       profile: getProfileOrNull(state),

src/profile-logic/data-structures.ts

@@ -348,8 +348,11 @@ export function getEmptySourceTable(): SourceTable {
     // If modifying this structure, please update all callers of this function to ensure
     // that they are pushing on correctly to the data structure. These pushes may not
     // be caught by the type system.
-    uuid: [],
+    id: [],
     filename: [],
+    startLine: [],
+    startColumn: [],
+    sourceMapURL: [],
     length: 0,
   };
 }

src/profile-logic/gecko-profile-versioning.ts

@@ -1522,6 +1522,41 @@ const _upgraders: {
     convertToVersion33Recursive(profile);
   },
 
+  [34]: (profile: any) => {
+    // The source table schema was updated:
+    // - The "uuid" field was renamed to "id".
+    // - startLine and startColumn were added, defaulting to 1 as they are 1-based.
+    // - sourceMapURL was added, defaulting to null.
+    function convertToVersion34Recursive(p: any) {
+      if (p.sources) {
+        const schema = p.sources.schema;
+
+        schema.id = schema.uuid;
+        delete schema.uuid;
+
+        // Set the source table schema indices for the new required fields.
+        // These indices specify which array position corresponds to each value.
+        // Then fill the arrays with default values.
+        // See the GeckoSourceTable type in src/types/gecko-profile.ts to see
+        // these values.
+        schema.startLine = 2;
+        schema.startColumn = 3;
+        schema.sourceMapURL = 4;
+
+        for (const row of p.sources.data) {
+          row[2] = 1;
+          row[3] = 1;
+          row[4] = null;
+        }
+      }
+
+      for (const subprocessProfile of p.processes) {
+        convertToVersion34Recursive(subprocessProfile);
+      }
+    }
+    convertToVersion34Recursive(profile);
+  },
+
   // If you add a new upgrader here, please document the change in
   // `docs-developer/CHANGELOG-formats.md`.
 };

src/profile-logic/global-data-collector.ts

@@ -61,7 +61,7 @@ export class GlobalDataCollector {
   _libNameToResourceIndex: Map<IndexIntoStringTable, IndexIntoResourceTable> =
     new Map();
   _originToResourceIndex: Map<string, IndexIntoResourceTable> = new Map();
-  _uuidToSourceIndex: Map<string, IndexIntoSourceTable> = new Map();
+  _idToSourceIndex: Map<string, IndexIntoSourceTable> = new Map();
   _filenameToSourceIndex: Map<IndexIntoStringTable, IndexIntoSourceTable> =
     new Map();
 
@@ -115,26 +115,39 @@ export class GlobalDataCollector {
 
   // Return the global index for this source, adding it to the global list if
   // necessary.
-  indexForSource(uuid: string | null, filename: string): IndexIntoSourceTable {
+  indexForSource(
+    id: string | null,
+    filename: string,
+    startLine: number = 1,
+    startColumn: number = 1,
+    sourceMapURL: string | null = null
+  ): IndexIntoSourceTable {
     let index: IndexIntoSourceTable | undefined;
 
-    if (uuid !== null) {
-      index = this._uuidToSourceIndex.get(uuid);
+    if (id !== null) {
+      index = this._idToSourceIndex.get(id);
     } else {
-      // For null UUIDs, use filename-based lookup
+      // For null IDs, use filename-based lookup.
       const filenameIndex = this._stringTable.indexForString(filename);
       index = this._filenameToSourceIndex.get(filenameIndex);
     }
 
     if (index === undefined) {
       index = this._sources.length;
       const filenameIndex = this._stringTable.indexForString(filename);
-      this._sources.uuid[index] = uuid;
+      const sourceMapURLIndex =
+        sourceMapURL !== null
+          ? this._stringTable.indexForString(sourceMapURL)
+          : null;
+      this._sources.id[index] = id;
       this._sources.filename[index] = filenameIndex;
+      this._sources.startLine[index] = startLine;
+      this._sources.startColumn[index] = startColumn;
+      this._sources.sourceMapURL[index] = sourceMapURLIndex;
       this._sources.length++;
 
-      if (uuid !== null) {
-        this._uuidToSourceIndex.set(uuid, index);
+      if (id !== null) {
+        this._idToSourceIndex.set(id, index);
       } else {
         this._filenameToSourceIndex.set(filenameIndex, index);
       }

src/profile-logic/merge-compare.ts

@@ -556,17 +556,26 @@ function mergeSources(
     const oldStringToNewStringPlusOne = translationMapsForStrings[profileIndex];
 
     for (let i = 0; i < sources.length; i++) {
-      const uuid = sources.uuid[i];
+      const id = sources.id[i];
       const originalUrlIndex = sources.filename[i];
       const newUrlIndex = oldStringToNewStringPlusOne[originalUrlIndex] - 1;
 
-      const sourceKey = uuid ?? `null-uuid-${newUrlIndex}`;
+      const originalSourceMapURLIndex = sources.sourceMapURL[i];
+      const newSourceMapURLIndex =
+        originalSourceMapURLIndex !== null
+          ? oldStringToNewStringPlusOne[originalSourceMapURLIndex] - 1
+          : null;
+
+      const sourceKey = id ?? `null-id-${newUrlIndex}`;
       let insertedSourceIndex = mapOfInsertedSources.get(sourceKey);
       if (insertedSourceIndex === undefined) {
         // Add new source
         insertedSourceIndex = newSources.length;
-        newSources.uuid[insertedSourceIndex] = uuid;
+        newSources.id[insertedSourceIndex] = id;
         newSources.filename[insertedSourceIndex] = newUrlIndex;
+        newSources.startLine[insertedSourceIndex] = sources.startLine[i];
+        newSources.startColumn[insertedSourceIndex] = sources.startColumn[i];
+        newSources.sourceMapURL[insertedSourceIndex] = newSourceMapURLIndex;
         newSources.length++;
         mapOfInsertedSources.set(sourceKey, insertedSourceIndex);
       }

src/profile-logic/process-profile.ts

@@ -431,13 +431,27 @@ function _extractJsFunction(
   let processedSourceIndex = null;
   if (sourceIndex !== undefined) {
     const geckoSourceIdx = parseInt(sourceIndex, 10);
-    // Look up the UUID for this source index from the process's sources table
+    // Look up the ID for this source index from the process's sources table.
     if (geckoSourceIdx < geckoSourceTable.data.length) {
-      const uuidIndex = geckoSourceTable.schema.uuid;
+      const idIndex = geckoSourceTable.schema.id;
       const filenameIndex = geckoSourceTable.schema.filename;
-      const uuid = geckoSourceTable.data[geckoSourceIdx][uuidIndex];
+      const startLineIndex = geckoSourceTable.schema.startLine;
+      const startColumnIndex = geckoSourceTable.schema.startColumn;
+      const sourceMapURLIndex = geckoSourceTable.schema.sourceMapURL;
+      const id = geckoSourceTable.data[geckoSourceIdx][idIndex];
       const filename = geckoSourceTable.data[geckoSourceIdx][filenameIndex];
-      processedSourceIndex = globalDataCollector.indexForSource(uuid, filename);
+      const startLine = geckoSourceTable.data[geckoSourceIdx][startLineIndex];
+      const startColumn =
+        geckoSourceTable.data[geckoSourceIdx][startColumnIndex];
+      const sourceMapURL =
+        geckoSourceTable.data[geckoSourceIdx][sourceMapURLIndex];
+      processedSourceIndex = globalDataCollector.indexForSource(
+        id,
+        filename,
+        startLine,
+        startColumn,
+        sourceMapURL
+      );
     }
   }
 

src/profile-logic/processed-profile-versioning.ts

@@ -2663,7 +2663,7 @@ const _upgraders: {
     // Create the sources table
     const sourceTable = {
       length: 0,
-      uuid: [] as Array<string | null>,
+      id: [] as Array<string | null>,
       filename: [] as Array<number>,
     };
 
@@ -2690,7 +2690,7 @@ const _upgraders: {
           if (sourceIndex === undefined) {
             // Add new entry to sources table
             sourceIndex = sourceTable.length;
-            sourceTable.uuid.push(null);
+            sourceTable.id.push(null);
             sourceTable.filename.push(fileNameIndex);
             sourceTable.length++;
             fileNameIndexToSourceIndex.set(fileNameIndex, sourceIndex);
@@ -3029,6 +3029,26 @@ const _upgraders: {
     profile.shared.nativeSymbols = newNativeSymbols;
     upgradeInfo.v60 = { threadMappings, newFuncCount: newFuncTable.length };
   },
+
+  [61]: (profile: any) => {
+    // The source table schema was updated:
+    // - The "uuid" field was renamed to "id". Profiles stored at v58-60 before
+    //   this rename may still have the old "uuid" field name.
+    // - startLine and startColumn were added, defaulting to 1 as they are 1-based.
+    // - sourceMapURL was added, defaulting to null.
+    const sources = profile.shared && profile.shared.sources;
+    if (sources) {
+      if (sources.uuid !== undefined && sources.id === undefined) {
+        sources.id = sources.uuid;
+        delete sources.uuid;
+      }
+      const length = sources.length;
+      sources.startLine = new Array(length).fill(1);
+      sources.startColumn = new Array(length).fill(1);
+      sources.sourceMapURL = new Array(length).fill(null);
+    }
+  },
+
   // If you add a new upgrader here, please document the change in
   // `docs-developer/CHANGELOG-formats.md`.
 };

src/profile-logic/profile-compacting.ts

@@ -746,6 +746,11 @@ function _gatherReferencesInSources(
     }
 
     referencedStrings[sources.filename[i]] = 1;
+
+    const sourceMapURL = sources.sourceMapURL[i];
+    if (sourceMapURL !== null) {
+      referencedStrings[sourceMapURL] = 1;
+    }
   }
 }
 
@@ -765,7 +770,18 @@ function _createCompactedSources(
     const newIndex = newSources.length++;
     newSources.filename[newIndex] =
       oldStringToNewStringPlusOne[sources.filename[i]] - 1;
-    newSources.uuid[newIndex] = sources.uuid[i];
+    newSources.id[newIndex] = sources.id[i];
+    newSources.startLine[newIndex] = sources.startLine[i];
+    newSources.startColumn[newIndex] = sources.startColumn[i];
+
+    // Translate the sourceMapURL string index if present.
+    // The old string indexes are no longer valid since we compacted the
+    // string array. All the indexes that reference them need to be updated.
+    const oldSourceMapURLIndex = sources.sourceMapURL[i];
+    newSources.sourceMapURL[newIndex] =
+      oldSourceMapURLIndex !== null
+        ? oldStringToNewStringPlusOne[oldSourceMapURLIndex] - 1
+        : null;
 
     oldSourceToNewSourcePlusOne[i] = newIndex + 1;
   }