Fix non-deterministic encoding of syntax contexts

compiler/rustc_span/src/hygiene.rs

@@ -1302,9 +1302,49 @@ pub struct HygieneEncodeContext {
     serialized_expns: Lock<FxHashSet<ExpnId>>,
 
     latest_expns: Lock<FxHashSet<ExpnId>>,
+
+    /// Maps every `SyntaxContext` into its encoding index.
+    /// Earlier the `ctxt.0` was used when writing metadata, however,
+    /// this results into non-deterministic metadata (see #129094).
+    /// The non-determinism is encountered when decoding syntax contexts
+    /// in `decode_syntax_context` function below. The syntax contexts from
+    /// other crate metadata can be decoded in different order, which results
+    /// into different ids assigned to decoded syntax contexts.
+    /// First invocation:
+    /// (ALLOC - syntax context id, ORIG - original id of decoded syntax context:
+    /// `raw_id` in `decode_syntax_context`)
+    /// ALLOC: #3, ORIG: 1
+    /// ALLOC: #9, ORIG: 18769
+    /// ALLOC: #10, ORIG: 25868
+    /// ALLOC: #11, ORIG: 18822
+    /// ALLOC: #12, ORIG: 23092
+    ///
+    /// Second invocation:
+    /// ALLOC: #3, ORIG: 1
+    /// ALLOC: #9, ORIG: 25868
+    /// ALLOC: #10, ORIG: 18769
+    /// ALLOC: #11, ORIG: 18822
+    /// ALLOC: #12, ORIG: 23092
+    ///
+    /// We see that `18769` and `25868` assigned different syntax context ids,
+    /// however, the order of encoding is deterministic, so we can remap allocated
+    /// syntax context ids into encoding indices and use them, thus outputting
+    /// same metadata.
+    encoding_indices: Lock<FxHashMap<SyntaxContext, u32>>,
 }
 
 impl HygieneEncodeContext {
+    fn get_encoding_index(&self, ctxt: SyntaxContext) -> u32 {
+        if ctxt.is_root() {
+            return 0;
+        }
+
+        let mut map = self.encoding_indices.lock();
+        // Zero is taken by root syntax context.
+        let encoding_index = map.len() + 1;
+        *map.entry(ctxt).or_insert(encoding_index as u32)
+    }
+
     /// Record the fact that we need to serialize the corresponding `ExpnData`.
     pub fn schedule_expn_data_for_encoding(&self, expn: ExpnId) {
         if !self.serialized_expns.lock().contains(&expn) {
@@ -1329,18 +1369,19 @@ impl HygieneEncodeContext {
 
             // Consume the current round of syntax contexts.
             // Drop the lock() temporary early.
-            // It's fine to iterate over a HashMap, because the serialization of the table
-            // that we insert data into doesn't depend on insertion order.
             #[allow(rustc::potential_query_instability)]
             let latest_ctxts = { mem::take(&mut *self.latest_ctxts.lock()) }.into_iter();
-            let all_ctxt_data: Vec<_> = HygieneData::with(|data| {
+            let mut all_ctxt_data: Vec<_> = HygieneData::with(|data| {
                 latest_ctxts
                     .map(|ctxt| (ctxt, data.syntax_context_data[ctxt.0 as usize].key()))
                     .collect()
             });
+
+            all_ctxt_data.sort_by_key(|&(ctxt, _)| self.get_encoding_index(ctxt));
+
             for (ctxt, ctxt_key) in all_ctxt_data {
                 if self.serialized_ctxts.lock().insert(ctxt) {
-                    encode_ctxt(encoder, ctxt.0, &ctxt_key);
+                    encode_ctxt(encoder, self.get_encoding_index(ctxt), &ctxt_key);
                 }
             }
 
@@ -1488,7 +1529,8 @@ pub fn raw_encode_syntax_context(
     if !context.serialized_ctxts.lock().contains(&ctxt) {
         context.latest_ctxts.lock().insert(ctxt);
     }
-    ctxt.0.encode(e);
+
+    context.get_encoding_index(ctxt).encode(e);
 }
 
 /// Updates the `disambiguator` field of the corresponding `ExpnData`

tests/run-make/parallel-reproducible-build/derives-issue-129094.rs

@@ -0,0 +1,5 @@
+#![crate_type = "lib"]
+#[derive(Clone, Copy, Hash, PartialEq, PartialOrd)]
+struct PackedPoint {
+    x: u32,
+}

tests/run-make/parallel-reproducible-build/rmake.rs

@@ -0,0 +1,42 @@
+//@ ignore-windows-gnu
+// GNU Linker for Windows is non-deterministic. (from `reproducible-build-2` test in this suite)
+
+use std::rc::Rc;
+
+use run_make_support::{bin_name, is_windows_msvc, rfs, run_in_tmpdir, rustc};
+
+/// Test that parallel compiler produces identical artifacts (binaries, metadata).
+fn main() {
+    const TESTS: &[(&str, &[&str])] = &[
+        ("static-muts-issue-140413", &["-Zthreads=50"]),
+        ("derives-issue-129094", &["-Zthreads=16", "-Copt-level=3"]),
+    ];
+
+    for (file, args) in TESTS {
+        let mut reference = None;
+        let bin_name = bin_name(file);
+
+        for _ in 0..100 {
+            // Tmp dir as previous runs affect output binary on windows.
+            run_in_tmpdir(|| {
+                let mut rustc = rustc();
+                rustc.input(format!("{file}.rs")).output(&bin_name);
+
+                for arg in *args {
+                    rustc.arg(arg);
+                }
+
+                if is_windows_msvc() {
+                    rustc.arg("-Clink-arg=/Brepro");
+                }
+
+                rustc.run();
+
+                let current = Rc::new(rfs::read(&bin_name));
+                reference.get_or_insert(Rc::clone(&current));
+
+                assert_eq!(Some(current), reference);
+            });
+        }
+    }
+}

tests/run-make/parallel-reproducible-build/static-muts-issue-140413.rs

@@ -0,0 +1,20 @@
+// Checks that mutable static items can have mutable slices and other references
+
+pub static mut TEST: &'static mut [isize] = &mut [1];
+pub static mut EMPTY: &'static mut [isize] = &mut [];
+pub static mut INT: &'static mut isize = &mut 1;
+
+// And the same for raw pointers.
+
+pub static mut TEST_RAW: *mut [isize] = &mut [1isize] as *mut _;
+pub static mut EMPTY_RAW: *mut [isize] = &mut [] as *mut _;
+pub static mut INT_RAW: *mut isize = &mut 1isize as *mut _;
+
+pub fn main() {
+    unsafe {
+        TEST[0] += 1;
+        assert_eq!(TEST[0], 2);
+        *INT_RAW += 1;
+        assert_eq!(*INT_RAW, 2);
+    }
+}