diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2025-12-27 23:21:13 +0100 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2025-12-27 23:21:13 +0100 |
| commit | 294ba569803972323a64670451a82af53c660541 (patch) | |
| tree | 1432420370a4676c985c5b9c06145f8a00223f88 | |
| parent | 7f920884cd004f9e2e60b3efda5bd75f287faa9d (diff) | |
Vendor import of llvm-project branch release/21.x llvmorg-21.1.7-0-gcd708029e0b2, a.k.a. 21.1.7 release.vendor/llvm-project/llvmorg-21.1.7-0-gcd708029e0b2
328 files changed, 5962 insertions, 2777 deletions
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h index c35311c88641..b929585205ae 100644 --- a/clang/include/clang-c/Index.h +++ b/clang/include/clang-c/Index.h @@ -6953,6 +6953,21 @@ clang_getCursorUnaryOperatorKind(CXCursor cursor); * @} */ +CINDEX_DEPRECATED +typedef void *CXRemapping; + +CINDEX_DEPRECATED CINDEX_LINKAGE CXRemapping clang_getRemappings(const char *); + +CINDEX_DEPRECATED CINDEX_LINKAGE CXRemapping +clang_getRemappingsFromFileList(const char **, unsigned); + +CINDEX_DEPRECATED CINDEX_LINKAGE unsigned clang_remap_getNumFiles(CXRemapping); + +CINDEX_DEPRECATED CINDEX_LINKAGE void +clang_remap_getFilenames(CXRemapping, unsigned, CXString *, CXString *); + +CINDEX_DEPRECATED CINDEX_LINKAGE void clang_remap_dispose(CXRemapping); + LLVM_CLANG_C_EXTERN_C_END #endif diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h index 9999a30c51ad..cb942ea865e2 100644 --- a/clang/include/clang/AST/APValue.h +++ b/clang/include/clang/AST/APValue.h @@ -143,7 +143,7 @@ public: AddrLabelDiff }; - class LValueBase { + class alignas(uint64_t) LValueBase { typedef llvm::PointerUnion<const ValueDecl *, const Expr *, TypeInfoLValue, DynamicAllocLValue> PtrTy; diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index 8c27728c404d..993eca8c4738 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -648,8 +648,7 @@ public: bool containsNonRelocatablePointerAuth(QualType T) { if (!isPointerAuthenticationAvailable()) return false; - return findPointerAuthContent(T) == - PointerAuthContent::AddressDiscriminatedData; + return findPointerAuthContent(T) != PointerAuthContent::None; } private: diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index fefdaba7f8bf..76747d2b1181 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -9417,9 +9417,9 @@ def NonStringDocs : Documentation { let Category = DocCatDecl; let Content = [{ The ``nonstring`` attribute can be applied to the declaration of a variable or -a field whose type is a character array to specify that the character array is -not intended to behave like a null-terminated string. This will silence -diagnostics with code like: +a field whose type is a character pointer or character array to specify that +the buffer is not intended to behave like a null-terminated string. This will +silence diagnostics with code like: .. code-block:: c diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td index a67b9995d3b5..4c7219c78c8b 100644 --- a/clang/include/clang/Basic/DiagnosticASTKinds.td +++ b/clang/include/clang/Basic/DiagnosticASTKinds.td @@ -507,6 +507,14 @@ def note_odr_number_of_bases : Note< "class has %0 base %plural{1:class|:classes}0">; def note_odr_enumerator : Note<"enumerator %0 with value %1 here">; def note_odr_missing_enumerator : Note<"no corresponding enumerator here">; +def note_odr_incompatible_fixed_underlying_type : Note< + "enumeration %0 declared with incompatible fixed underlying types (%1 vs. " + "%2)">; +def note_odr_fixed_underlying_type : Note< + "enumeration %0 has fixed underlying type here">; +def note_odr_missing_fixed_underlying_type : Note< + "enumeration %0 missing fixed underlying type here">; + def err_odr_field_type_inconsistent : Error< "field %0 declared with incompatible types in different " "translation units (%1 vs. %2)">; diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index c28a919e35d0..76f9addab47d 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -624,6 +624,7 @@ def MissingFieldInitializers : DiagGroup<"missing-field-initializers", def ModuleLock : DiagGroup<"module-lock">; def ModuleBuild : DiagGroup<"module-build">; def ModuleImport : DiagGroup<"module-import">; +def ModuleValidation : DiagGroup<"module-validation">; def ModuleConflict : DiagGroup<"module-conflict">; def ModuleFileExtension : DiagGroup<"module-file-extension">; def ModuleIncludeDirectiveTranslation : DiagGroup<"module-include-translation">; diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td index 584c8d62280b..6494f3415b7a 100644 --- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td +++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td @@ -82,6 +82,10 @@ def remark_module_import : Remark< "importing module '%0'%select{| into '%3'}2 from '%1'">, ShowInSystemHeader, InGroup<ModuleImport>; +def remark_module_validation : Remark< + "validating %0 input files in module '%1' from '%2'">, + ShowInSystemHeader, + InGroup<ModuleValidation>; def err_imported_module_not_found : Error< "module '%0' in precompiled file '%1' %select{(imported by precompiled file '%2') |}4" diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index 72f23614aef1..b040181beaff 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -147,14 +147,17 @@ FEATURE(type_sanitizer, LangOpts.Sanitize.has(SanitizerKind::Type)) FEATURE(thread_sanitizer, LangOpts.Sanitize.has(SanitizerKind::Thread)) FEATURE(dataflow_sanitizer, LangOpts.Sanitize.has(SanitizerKind::DataFlow)) FEATURE(scudo, LangOpts.Sanitize.hasOneOf(SanitizerKind::Scudo)) -FEATURE(ptrauth_intrinsics, LangOpts.PointerAuthIntrinsics) -EXTENSION(ptrauth_qualifier, LangOpts.PointerAuthIntrinsics) +FEATURE(ptrauth_intrinsics, LangOpts.PointerAuthIntrinsics && + PP.getTargetInfo().getTriple().isOSDarwin()) +FEATURE(ptrauth_qualifier, LangOpts.PointerAuthIntrinsics && + PP.getTargetInfo().getTriple().isOSDarwin()) FEATURE(ptrauth_calls, LangOpts.PointerAuthCalls) FEATURE(ptrauth_returns, LangOpts.PointerAuthReturns) FEATURE(ptrauth_vtable_pointer_address_discrimination, LangOpts.PointerAuthVTPtrAddressDiscrimination) FEATURE(ptrauth_vtable_pointer_type_discrimination, LangOpts.PointerAuthVTPtrTypeDiscrimination) FEATURE(ptrauth_type_info_vtable_pointer_discrimination, LangOpts.PointerAuthTypeInfoVTPtrDiscrimination) FEATURE(ptrauth_member_function_pointer_type_discrimination, LangOpts.PointerAuthCalls) +FEATURE(ptrauth_signed_block_descriptors, LangOpts.PointerAuthBlockDescriptorPointers) FEATURE(ptrauth_function_pointer_type_discrimination, LangOpts.PointerAuthFunctionTypeDiscrimination) FEATURE(ptrauth_indirect_gotos, LangOpts.PointerAuthIndirectGotos) FEATURE(ptrauth_init_fini, LangOpts.PointerAuthInitFini) @@ -163,7 +166,7 @@ FEATURE(ptrauth_elf_got, LangOpts.PointerAuthELFGOT) FEATURE(ptrauth_objc_isa, LangOpts.PointerAuthObjcIsa) FEATURE(ptrauth_objc_interface_sel, LangOpts.PointerAuthObjcInterfaceSel) -FEATURE(ptrauth_objc_signable_class, true) +FEATURE(ptrauth_objc_signable_class, LangOpts.PointerAuthIntrinsics) FEATURE(ptrauth_objc_method_list_pointer, LangOpts.PointerAuthCalls) EXTENSION(swiftcc, diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index 0d546cb3b847..25f4575a5425 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -138,6 +138,8 @@ LANGOPT(PointerAuthObjcInterfaceSel, 1, 0, NotCompatible, "authentication of SEL LANGOPT(PointerAuthObjcInterfaceSelKey, 16, 0, NotCompatible, "authentication key for SEL fields of ObjC interfaces") LANGOPT(PointerAuthObjcClassROPointers, 1, 0, Benign, "class_ro_t pointer authentication") +LANGOPT(PointerAuthBlockDescriptorPointers, 1, 0, NotCompatible, "enable signed block descriptors") + LANGOPT(DoubleSquareBracketAttributes, 1, 0, NotCompatible, "'[[]]' attributes extension for all language standard modes") LANGOPT(ExperimentalLateParseAttributes, 1, 0, NotCompatible, "experimental late parsing of attributes") @@ -496,6 +498,8 @@ LANGOPT(CheckConstexprFunctionBodies, 1, 1, Benign, LANGOPT(BoundsSafety, 1, 0, NotCompatible, "Bounds safety extension for C") +LANGOPT(EnableLifetimeSafety, 1, 0, NotCompatible, "Experimental lifetime safety analysis for C++") + LANGOPT(PreserveVec3Type, 1, 0, NotCompatible, "Preserve 3-component vector type") #undef LANGOPT diff --git a/clang/include/clang/Basic/PointerAuthOptions.h b/clang/include/clang/Basic/PointerAuthOptions.h index fb6dddf3ae9c..2b920250721f 100644 --- a/clang/include/clang/Basic/PointerAuthOptions.h +++ b/clang/include/clang/Basic/PointerAuthOptions.h @@ -23,6 +23,10 @@ namespace clang { +/// Constant discriminator to be used with block descriptor pointers. The value +/// is ptrauth_string_discriminator("block_descriptor") +constexpr uint16_t BlockDescriptorConstantDiscriminator = 0xC0BB; + /// Constant discriminator to be used with function pointers in .init_array and /// .fini_array. The value is ptrauth_string_discriminator("init_fini") constexpr uint16_t InitFiniPointerConstantDiscriminator = 0xD9D4; @@ -223,6 +227,18 @@ struct PointerAuthOptions { /// The ABI for function addresses in .init_array and .fini_array PointerAuthSchema InitFiniPointers; + /// The ABI for block invocation function pointers. + PointerAuthSchema BlockInvocationFunctionPointers; + + /// The ABI for block object copy/destroy function pointers. + PointerAuthSchema BlockHelperFunctionPointers; + + /// The ABI for __block variable copy/destroy function pointers. + PointerAuthSchema BlockByrefHelperFunctionPointers; + + /// The ABI for pointers to block descriptors. + PointerAuthSchema BlockDescriptorPointers; + /// The ABI for Objective-C method lists. PointerAuthSchema ObjCMethodListFunctionPointers; diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td index 275bb2b9924d..cc5ab38f8f96 100644 --- a/clang/include/clang/Basic/riscv_vector.td +++ b/clang/include/clang/Basic/riscv_vector.td @@ -56,34 +56,8 @@ multiclass RVVVLEFFBuiltin<list<string> types> { SupportOverloading = false, UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - // Move mask to right before vl. - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - IntrinsicTypes = {ResultType, Ops[4]->getType(), Ops[2]->getType()}; - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - IntrinsicTypes = {ResultType, Ops[3]->getType(), Ops[1]->getType()}; - } - Value *NewVL = Ops[2]; - Ops.erase(Ops.begin() + 2); - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); - llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); - // Store new_vl. - clang::CharUnits Align; - if (IsMasked) - Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType()); - else - Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); - Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); - return V; - } + return emitRVVVLEFFBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach type = types in { def : RVVBuiltin<"v", "vPCePz", type>; @@ -139,17 +113,8 @@ multiclass RVVIndexedLoad<string op> { let HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - if (IsMasked) { - // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) - std::swap(Ops[0], Ops[2]); - } else { - // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) - std::swap(Ops[0], Ops[1]); - } - if (IsMasked) - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; - else - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()}; + return emitRVVVSEMaskBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { let Name = "vsm_v"; @@ -177,17 +142,8 @@ multiclass RVVVSSEBuiltin<list<string> types> { HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - if (IsMasked) { - // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) - std::swap(Ops[0], Ops[3]); - } else { - // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); - } - if (IsMasked) - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[4]->getType()}; - else - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; + return emitRVVVSSEBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach type = types in { def : RVVBuiltin<"v", "0Petv", type>; @@ -202,17 +158,8 @@ multiclass RVVIndexedStore<string op> { let HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - if (IsMasked) { - // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) - std::swap(Ops[0], Ops[3]); - } else { - // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); - } - if (IsMasked) - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), Ops[4]->getType()}; - else - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), Ops[3]->getType()}; + return emitRVVIndexedStoreBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach type = TypeList in { foreach eew_list = EEWList[0-2] in { @@ -367,28 +314,8 @@ multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, op2, mask, vl, policy - IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; - } else { - // passthru, op1, op2, vl - IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoUnaryBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"v", "vv", type_range>; } @@ -400,32 +327,8 @@ multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); - Ops.insert(Ops.begin() + 2, - llvm::Constant::getAllOnesValue(ElemTy)); - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, po2, mask, vl, policy - IntrinsicTypes = {ResultType, - ElemTy, - Ops[4]->getType()}; - } else { - // passthru, op1, op2, vl - IntrinsicTypes = {ResultType, - ElemTy, - Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoVNotBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"v", "vv", type_range>; def : RVVBuiltin<"Uv", "UvUv", type_range>; @@ -437,13 +340,8 @@ multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { IRName = IR, HasMasked = false, ManualCodegen = [{ - { - // op1, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType()}; - Ops.insert(Ops.begin() + 1, Ops[0]); - break; - } + return emitRVVPseudoMaskBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"m", "mm", type_range>; } @@ -455,28 +353,8 @@ multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.insert(Ops.begin() + 2, Ops[1]); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, op2, mask, vl - IntrinsicTypes = {ResultType, - Ops[2]->getType(), - Ops.back()->getType()}; - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - // op1, po2, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), Ops[2]->getType()}; - Ops.insert(Ops.begin() + 2, Ops[1]); - break; - } - break; - } + return emitRVVPseudoVFUnaryBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"v", "vv", type_range>; } @@ -490,33 +368,8 @@ multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - auto ElemTy = cast<llvm::VectorType>(Ops[1]->getType())->getElementType(); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, op2, mask, vl, policy - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - ElemTy, - Ops[4]->getType()}; - } else { - // passtru, op1, op2, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - ElemTy, - Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoVWCVTBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach s_p = suffixes_prototypes in { def : RVVBuiltin<s_p[0], s_p[1], type_range>; @@ -532,32 +385,8 @@ multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, xlen, mask, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - Ops[4]->getType(), - Ops[4]->getType()}; - } else { - // passthru, op1, xlen, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - Ops[3]->getType(), - Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoVNCVTBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach s_p = suffixes_prototypes in { def : RVVBuiltin<s_p[0], s_p[1], type_range>; @@ -575,17 +404,8 @@ let HasBuiltinAlias = false, HasVL = false, HasMasked = false, UnMaskedPolicyScheme = NonePolicy, MaskedPolicyScheme = NonePolicy, Log2LMUL = [0], IRName = "", ManualCodegen = [{ - { - LLVMContext &Context = CGM.getLLVMContext(); - llvm::MDBuilder MDHelper(Context); - - llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "vlenb")}; - llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); - llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); - llvm::Function *F = - CGM.getIntrinsic(llvm::Intrinsic::read_register, {SizeTy}); - return Builder.CreateCall(F, Metadata); - } + return emitRVVVlenbBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { def vlenb : RVVBuiltin<"", "u", "i">; @@ -660,7 +480,10 @@ let HasBuiltinAlias = false, HasMasked = false, MaskedPolicyScheme = NonePolicy, Log2LMUL = [0], - ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type + ManualCodegen = [{ + return emitRVVVsetvliBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); + }] in // Set XLEN type { def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; @@ -720,43 +543,10 @@ multiclass RVVUnitStridedSegLoadTuple<string op> { MaskedIRName = op # nf # "_mask", NF = nf, ManualCodegen = [{ - { - SmallVector<llvm::Value*, 6> Operands; - - bool NoPassthru = - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | - (!IsMasked && (PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; - - if (IsMasked) - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[0]->getType(), Ops.back()->getType()}; - else - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType()}; - - if (NoPassthru) { // Push poison into passthru - Operands.push_back(llvm::PoisonValue::get(ResultType)); - } else { // Push intrinsics operands into passthru - llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; - Operands.push_back(PassthruOperand); - } - - Operands.push_back(Ops[Offset]); // Ptr - if (IsMasked) - Operands.push_back(Ops[0]); - Operands.push_back(Ops[Offset + 1]); // VL - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - if (ReturnValue.isNull()) - return LoadValue; - else - return Builder.CreateStore(LoadValue, ReturnValue.getValue()); - } - }] in { + return emitRVVUnitStridedSegLoadTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, + IsMasked, SegInstSEW); + }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", T # "vPCe", type>; if !not(IsFloat<type>.val) then { @@ -784,31 +574,10 @@ multiclass RVVUnitStridedSegStoreTuple<string op> { NF = nf, HasMaskedOffOperand = false, ManualCodegen = [{ - { - // Masked - // Builtin: (mask, ptr, v_tuple, vl) - // Intrinsic: (tuple, ptr, mask, vl) - // Unmasked - // Builtin: (ptr, v_tuple, vl) - // Intrinsic: (tuple, ptr, vl) - unsigned Offset = IsMasked ? 1 : 0; - - SmallVector<llvm::Value*, 5> Operands; - Operands.push_back(Ops[Offset + 1]); // tuple - Operands.push_back(Ops[Offset]); // Ptr - if (IsMasked) - Operands.push_back(Ops[0]); - Operands.push_back(Ops[Offset + 2]); // VL - Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - if (IsMasked) - IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Ops[0]->getType(), Operands.back()->getType()}; - else - IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Operands.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); - } - }] in { + return emitRVVUnitStridedSegStoreTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, + IsMasked, SegInstSEW); + }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", "0Pe" # T # "v", type>; if !not(IsFloat<type>.val) then { @@ -835,52 +604,9 @@ multiclass RVVUnitStridedSegLoadFFTuple<string op> { MaskedIRName = op # nf # "ff_mask", NF = nf, ManualCodegen = [{ - { - SmallVector<llvm::Value*, 6> Operands; - - bool NoPassthru = - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | - (!IsMasked && (PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; - - if (IsMasked) - IntrinsicTypes = {ResultType, Ops.back()->getType(), Ops[Offset]->getType(), Ops[0]->getType()}; - else - IntrinsicTypes = {ResultType, Ops.back()->getType(), Ops[Offset]->getType()}; - - if (NoPassthru) { // Push poison into passthru - Operands.push_back(llvm::PoisonValue::get(ResultType)); - } else { // Push intrinsics operands into passthru - llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; - Operands.push_back(PassthruOperand); - } - - Operands.push_back(Ops[Offset]); // Ptr - if (IsMasked) - Operands.push_back(Ops[0]); - Operands.push_back(Ops[Offset + 2]); // vl - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - // Get alignment from the new vl operand - clang::CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType()); - - llvm::Value *ReturnTuple = Builder.CreateExtractValue(LoadValue, 0); - - // Store new_vl - llvm::Value *V = Builder.CreateExtractValue(LoadValue, 1); - Builder.CreateStore(V, Address(Ops[Offset + 1], V->getType(), Align)); - - if (ReturnValue.isNull()) - return ReturnTuple; - else - return Builder.CreateStore(ReturnTuple, ReturnValue.getValue()); - } + return emitRVVUnitStridedSegLoadFFTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", T # "vPCePz", type>; @@ -908,43 +634,9 @@ multiclass RVVStridedSegLoadTuple<string op> { MaskedIRName = op # nf # "_mask", NF = nf, ManualCodegen = [{ - { - SmallVector<llvm::Value*, 7> Operands; - - bool NoPassthru = - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | - (!IsMasked && (PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; - - if (IsMasked) - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType(), Ops[0]->getType()}; - else - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType()}; - - if (NoPassthru) { // Push poison into passthru - Operands.push_back(llvm::PoisonValue::get(ResultType)); - } else { // Push intrinsics operands into passthru - llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; - Operands.push_back(PassthruOperand); - } - - Operands.push_back(Ops[Offset]); // Ptr - Operands.push_back(Ops[Offset + 1]); // Stride - if (IsMasked) - Operands.push_back(Ops[0]); - Operands.push_back(Ops[Offset + 2]); // VL - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - - if (ReturnValue.isNull()) - return LoadValue; - else - return Builder.CreateStore(LoadValue, ReturnValue.getValue()); - } + return emitRVVStridedSegLoadTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", T # "vPCet", type>; @@ -974,31 +666,9 @@ multiclass RVVStridedSegStoreTuple<string op> { HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - { - // Masked - // Builtin: (mask, ptr, stride, v_tuple, vl) - // Intrinsic: (tuple, ptr, stride, mask, vl) - // Unmasked - // Builtin: (ptr, stride, v_tuple, vl) - // Intrinsic: (tuple, ptr, stride, vl) - unsigned Offset = IsMasked ? 1 : 0; - - SmallVector<llvm::Value*, 6> Operands; - Operands.push_back(Ops[Offset + 2]); // tuple - Operands.push_back(Ops[Offset]); // Ptr - Operands.push_back(Ops[Offset + 1]); // Stride - if (IsMasked) - Operands.push_back(Ops[0]); - Operands.push_back(Ops[Offset + 3]); // VL - Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - if (IsMasked) - IntrinsicTypes = {Operands[0]->getType(), Operands[1]->getType(), Operands.back()->getType(), Ops[0]->getType()}; - else - IntrinsicTypes = {Operands[0]->getType(), Operands[1]->getType(), Operands.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); - } + return emitRVVStridedSegStoreTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", "0Pet" # T # "v", type>; @@ -1021,47 +691,9 @@ multiclass RVVIndexedSegLoadTuple<string op> { MaskedIRName = op # nf # "_mask", NF = nf, ManualCodegen = [{ - { - SmallVector<llvm::Value*, 7> Operands; - - bool NoPassthru = - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | - (!IsMasked && (PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; - - if (NoPassthru) { // Push poison into passthru - Operands.push_back(llvm::PoisonValue::get(ResultType)); - } else { // Push intrinsics operands into passthru - llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; - Operands.push_back(PassthruOperand); - } - - Operands.push_back(Ops[Offset]); // Ptr - Operands.push_back(Ops[Offset + 1]); // Idx - if (IsMasked) - Operands.push_back(Ops[0]); - Operands.push_back(Ops[Offset + 2]); // VL - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - if (IsMasked) - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), - Ops[Offset + 1]->getType(), - Ops[0]->getType(), - Ops.back()->getType()}; - else - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), - Ops[Offset + 1]->getType(), - Ops.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - - if (ReturnValue.isNull()) - return LoadValue; - else - return Builder.CreateStore(LoadValue, ReturnValue.getValue()); - } + return emitRVVIndexedSegLoadTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", T # "vPCe" # eew_type # "Uv", type>; @@ -1087,34 +719,9 @@ multiclass RVVIndexedSegStoreTuple<string op> { HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - { - // Masked - // Builtin: (mask, ptr, index, v_tuple, vl) - // Intrinsic: (tuple, ptr, index, mask, vl) - // Unmasked - // Builtin: (ptr, index, v_tuple, vl) - // Intrinsic: (tuple, ptr, index, vl) - unsigned Offset = IsMasked ? 1 : 0; - - SmallVector<llvm::Value*, 6> Operands; - Operands.push_back(Ops[Offset + 2]); // tuple - Operands.push_back(Ops[Offset]); // Ptr - Operands.push_back(Ops[Offset + 1]); // Idx - if (IsMasked) - Operands.push_back(Ops[0]); - Operands.push_back(Ops[Offset + 3]); // VL - Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - if (IsMasked) - IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Ops[Offset + 1]->getType(), - Ops[0]->getType(), - Operands.back()->getType()}; - else - IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Ops[Offset + 1]->getType(), - Operands.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); - } + return emitRVVIndexedSegStoreTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", "0Pe" # eew_type # "Uv" # T # "v", type>; @@ -1355,37 +962,8 @@ defm vssub : RVVSignedBinBuiltinSet; let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, op1, round_mode, vl) - // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) - - SmallVector<llvm::Value*, 7> Operands; - bool HasMaskedOff = !( - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || - (!IsMasked && PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? - (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); - - if (!HasMaskedOff) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - else - Operands.push_back(Ops[IsMasked ? 1 : 0]); - - Operands.push_back(Ops[Offset]); // op0 - Operands.push_back(Ops[Offset + 1]); // op1 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - Operands.push_back(Ops[Offset + 2]); // vxrm - Operands.push_back(Ops[Offset + 3]); // vl - - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), Ops.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); + return emitRVVAveragingBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); } }] in { // 12.2. Vector Single-Width Averaging Add and Subtract @@ -1404,38 +982,8 @@ let ManualCodegen = [{ let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, op1, round_mode, vl) - // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) - - SmallVector<llvm::Value*, 7> Operands; - bool HasMaskedOff = !( - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || - (!IsMasked && PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? - (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); - - if (!HasMaskedOff) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - else - Operands.push_back(Ops[IsMasked ? 1 : 0]); - - Operands.push_back(Ops[Offset]); // op0 - Operands.push_back(Ops[Offset + 1]); // op1 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - Operands.push_back(Ops[Offset + 2]); // vxrm - Operands.push_back(Ops[Offset + 3]); // vl - - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), - Ops.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); + return emitRVVNarrowingClipBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); } }] in { // 12.5. Vector Narrowing Fixed-Point Clip Instructions @@ -1459,47 +1007,8 @@ enum __RISCV_FRM { let UnMaskedPolicyScheme = HasPassthruOperand in { let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, op1, round_mode, vl) - // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) - - SmallVector<llvm::Value*, 7> Operands; - bool HasMaskedOff = !( - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || - (!IsMasked && PolicyAttrs & RVV_VTA)); - bool HasRoundModeOp = IsMasked ? - (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : - (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); - - unsigned Offset = IsMasked ? - (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); - - if (!HasMaskedOff) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - else - Operands.push_back(Ops[IsMasked ? 1 : 0]); - - Operands.push_back(Ops[Offset]); // op0 - Operands.push_back(Ops[Offset + 1]); // op1 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - if (HasRoundModeOp) { - Operands.push_back(Ops[Offset + 2]); // frm - Operands.push_back(Ops[Offset + 3]); // vl - } else { - Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm - Operands.push_back(Ops[Offset + 2]); // vl - } - - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), - Operands.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); + return emitRVVFloatingPointBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); } }] in { let HasFRMRoundModeOp = true in { @@ -1536,47 +1045,9 @@ let ManualCodegen = [{ let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, op1, round_mode, vl) - // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) - - SmallVector<llvm::Value*, 7> Operands; - bool HasMaskedOff = !( - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || - (!IsMasked && PolicyAttrs & RVV_VTA)); - bool HasRoundModeOp = IsMasked ? - (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : - (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); - - unsigned Offset = IsMasked ? - (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); - - if (!HasMaskedOff) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - else - Operands.push_back(Ops[IsMasked ? 1 : 0]); - - Operands.push_back(Ops[Offset]); // op0 - Operands.push_back(Ops[Offset + 1]); // op1 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - if (HasRoundModeOp) { - Operands.push_back(Ops[Offset + 2]); // frm - Operands.push_back(Ops[Offset + 3]); // vl - } else { - Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm - Operands.push_back(Ops[Offset + 2]); // vl - } - - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), - Ops.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); + return emitRVVWideningFloatingPointBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); } }] in { let HasFRMRoundModeOp = true in { @@ -1618,39 +1089,8 @@ let ManualCodegen = [{ let UnMaskedPolicyScheme = HasPolicyOperand in { let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, op1, round_mode, vl) - // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) - - SmallVector<llvm::Value*, 7> Operands; - bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; - - unsigned Offset = IsMasked ? 2 : 1; - - Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough - - Operands.push_back(Ops[Offset]); // op0 - Operands.push_back(Ops[Offset + 1]); // op1 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - if (HasRoundModeOp) { - Operands.push_back(Ops[Offset + 2]); // frm - Operands.push_back(Ops[Offset + 3]); // vl - } else { - Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm - Operands.push_back(Ops[Offset + 2]); // vl - } - - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), - Operands.back()->getType()}; - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - - return Builder.CreateCall(F, Operands, ""); + return emitRVVFMABuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); } }] in { let HasFRMRoundModeOp = 1 in { @@ -1677,39 +1117,8 @@ let ManualCodegen = [{ let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, op1, round_mode, vl) - // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) - - SmallVector<llvm::Value*, 7> Operands; - bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; - - unsigned Offset = IsMasked ? 2 : 1; - - Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough - - Operands.push_back(Ops[Offset]); // op0 - Operands.push_back(Ops[Offset + 1]); // op1 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - if (HasRoundModeOp) { - Operands.push_back(Ops[Offset + 2]); // frm - Operands.push_back(Ops[Offset + 3]); // vl - } else { - Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm - Operands.push_back(Ops[Offset + 2]); // vl - } - - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), - Operands.back()->getType()}; - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - - return Builder.CreateCall(F, Operands, ""); + return emitRVVWideningFMABuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); } }] in { let HasFRMRoundModeOp = 1 in { @@ -1747,45 +1156,8 @@ let ManualCodegen = [{ let UnMaskedPolicyScheme = HasPassthruOperand in { let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, round_mode, vl) - // Masked: (passthru, op0, mask, frm, vl, policy) - - SmallVector<llvm::Value*, 6> Operands; - bool HasMaskedOff = !( - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || - (!IsMasked && PolicyAttrs & RVV_VTA)); - bool HasRoundModeOp = IsMasked ? - (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : - (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); - - unsigned Offset = IsMasked ? - (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); - - if (!HasMaskedOff) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - else - Operands.push_back(Ops[IsMasked ? 1 : 0]); - - Operands.push_back(Ops[Offset]); // op0 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - if (HasRoundModeOp) { - Operands.push_back(Ops[Offset + 1]); // frm - Operands.push_back(Ops[Offset + 2]); // vl - } else { - Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm - Operands.push_back(Ops[Offset + 1]); // vl - } - - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Operands.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); + return emitRVVFloatingUnaryBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); } }] in { let HasFRMRoundModeOp = 1 in { @@ -1947,45 +1319,8 @@ def vfwcvtbf16_f_f_v : RVVConvBuiltin<"Fw", "Fwv", "y", "vfwcvtbf16_f">; let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, frm, vl) - // Masked: (passthru, op0, mask, frm, vl, policy) - SmallVector<llvm::Value*, 6> Operands; - bool HasMaskedOff = !( - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || - (!IsMasked && PolicyAttrs & RVV_VTA)); - bool HasRoundModeOp = IsMasked ? - (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : - (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); - - unsigned Offset = IsMasked ? - (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); - - if (!HasMaskedOff) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - else - Operands.push_back(Ops[IsMasked ? 1 : 0]); - - Operands.push_back(Ops[Offset]); // op0 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - if (HasRoundModeOp) { - Operands.push_back(Ops[Offset + 1]); // frm - Operands.push_back(Ops[Offset + 2]); // vl - } else { - Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm - Operands.push_back(Ops[Offset + 1]); // vl - } - - if (IsMasked) - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), - Operands.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); + return emitRVVFloatingConvBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); } }] in { let HasFRMRoundModeOp = 1 in { @@ -2151,44 +1486,9 @@ defm vfredmax : RVVFloatingReductionBuiltin; defm vfredmin : RVVFloatingReductionBuiltin; let ManualCodegen = [{ { - // LLVM intrinsic - // Unmasked: (passthru, op0, op1, round_mode, vl) - // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) - - SmallVector<llvm::Value*, 6> Operands; - bool HasMaskedOff = !( - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || - (!IsMasked && PolicyAttrs & RVV_VTA)); - bool HasRoundModeOp = IsMasked ? - (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : - (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); - - unsigned Offset = IsMasked ? - (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); - - if (!HasMaskedOff) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - else - Operands.push_back(Ops[IsMasked ? 1 : 0]); - - Operands.push_back(Ops[Offset]); // op0 - Operands.push_back(Ops[Offset + 1]); // op1 - - if (IsMasked) - Operands.push_back(Ops[0]); // mask - - if (HasRoundModeOp) { - Operands.push_back(Ops[Offset + 2]); // frm - Operands.push_back(Ops[Offset + 3]); // vl - } else { - Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm - Operands.push_back(Ops[Offset + 2]); // vl - } - - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), - Ops.back()->getType()}; - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - return Builder.CreateCall(F, Operands, ""); + return emitRVVFloatingReductionBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); } }] in { let HasFRMRoundModeOp = 1 in { @@ -2346,37 +1646,8 @@ let HasMasked = false, let HasMasked = false, HasVL = false, IRName = "" in { let Name = "vreinterpret_v", MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - if (ResultType->isIntOrIntVectorTy(1) || - Ops[0]->getType()->isIntOrIntVectorTy(1)) { - assert(isa<ScalableVectorType>(ResultType) && - isa<ScalableVectorType>(Ops[0]->getType())); - - LLVMContext &Context = CGM.getLLVMContext(); - ScalableVectorType *Boolean64Ty = - ScalableVectorType::get(llvm::Type::getInt1Ty(Context), 64); - - if (ResultType->isIntOrIntVectorTy(1)) { - // Casting from m1 vector integer -> vector boolean - // Ex: <vscale x 8 x i8> - // --(bitcast)--------> <vscale x 64 x i1> - // --(vector_extract)-> <vscale x 8 x i1> - llvm::Value *BitCast = Builder.CreateBitCast(Ops[0], Boolean64Ty); - return Builder.CreateExtractVector(ResultType, BitCast, - ConstantInt::get(Int64Ty, 0)); - } else { - // Casting from vector boolean -> m1 vector integer - // Ex: <vscale x 1 x i1> - // --(vector_insert)-> <vscale x 64 x i1> - // --(bitcast)-------> <vscale x 8 x i8> - llvm::Value *Boolean64Val = - Builder.CreateInsertVector(Boolean64Ty, - llvm::PoisonValue::get(Boolean64Ty), - Ops[0], - ConstantInt::get(Int64Ty, 0)); - return Builder.CreateBitCast(Boolean64Val, ResultType); - } - } - return Builder.CreateBitCast(Ops[0], ResultType); + return emitRVVReinterpretBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { // Reinterpret between different type under the same SEW and LMUL def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; @@ -2502,25 +1773,8 @@ let HasMasked = false, HasVL = false, IRName = "" in { let Name = "vget_v", MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - { - auto *VecTy = cast<ScalableVectorType>(ResultType); - if (auto *OpVecTy = dyn_cast<ScalableVectorType>(Ops[0]->getType())) { - unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); - assert(isPowerOf2_32(MaxIndex)); - // Mask to only valid indices. - Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); - Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); - Ops[1] = Builder.CreateMul(Ops[1], - ConstantInt::get(Ops[1]->getType(), - VecTy->getMinNumElements())); - return Builder.CreateExtractVector(ResultType, Ops[0], Ops[1]); - } - - return Builder.CreateIntrinsic(Intrinsic::riscv_tuple_extract, - {ResultType, Ops[0]->getType()}, - {Ops[0], Builder.CreateTrunc(Ops[1], - Builder.getInt32Ty())}); - } + return emitRVVGetBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfdy", dst_lmul # "v">; @@ -2535,25 +1789,8 @@ let HasMasked = false, HasVL = false, IRName = "" in { let Name = "vset_v", MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - { - if (auto *ResVecTy = dyn_cast<ScalableVectorType>(ResultType)) { - auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); - unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); - assert(isPowerOf2_32(MaxIndex)); - // Mask to only valid indices. - Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); - Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); - Ops[1] = Builder.CreateMul(Ops[1], - ConstantInt::get(Ops[1]->getType(), - VecTy->getMinNumElements())); - return Builder.CreateInsertVector(ResultType, Ops[0], Ops[2], Ops[1]); - } - - return Builder.CreateIntrinsic(Intrinsic::riscv_tuple_insert, - {ResultType, Ops[2]->getType()}, - {Ops[0], Ops[2], - Builder.CreateTrunc(Ops[1],Builder.getInt32Ty())}); - } + return emitRVVSetBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfdy">; @@ -2571,26 +1808,8 @@ let HasMasked = false, HasVL = false, IRName = "" in { MaskedPolicyScheme = NonePolicy, SupportOverloading = false, ManualCodegen = [{ - { - llvm::Value *ReturnVector = llvm::PoisonValue::get(ResultType); - auto *VecTy = cast<ScalableVectorType>(Ops[0]->getType()); - for (unsigned I = 0, N = Ops.size(); I < N; ++I) { - if (isa<ScalableVectorType>(ResultType)) { - llvm::Value *Idx = ConstantInt::get(Builder.getInt64Ty(), - VecTy->getMinNumElements() * I); - ReturnVector = - Builder.CreateInsertVector(ResultType, ReturnVector, Ops[I], Idx); - } else { - llvm::Value *Idx = ConstantInt::get(Builder.getInt32Ty(), I); - ReturnVector = - Builder.CreateIntrinsic(Intrinsic::riscv_tuple_insert, - {ResultType, Ops[I]->getType()}, - {ReturnVector, Ops[I], Idx}); - } - - } - return ReturnVector; - } + return emitRVVCreateBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { // Since the vcreate_v uses LFixedLog2LMUL, setting the Log2LMUL to [-3] can diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index e7c7e9d93fe2..958d0d05ade2 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -990,6 +990,13 @@ def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>, Visibility<[ClangOption, CLOption, FlangOption]>, HelpText<"Pass <arg> to the linker">, MetaVarName<"<arg>">, Group<Link_Group>; +def Xthinlto_distributor_EQ : CommaJoined<["-"], "Xthinlto-distributor=">, + Flags<[LinkOption]>, + Visibility<[ClangOption, CLOption]>, + HelpText<"Pass <arg> to the ThinLTO distributor process. Can be specified " + "multiple times or with comma-separated values.">, + MetaVarName<"<arg>">, + Group<Link_Group>; def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">, Visibility<[ClangOption, FlangOption]>, HelpText<"Pass <arg> to the offload linkers or the ones identified by -<triple>">, @@ -1910,6 +1917,14 @@ defm bounds_safety : BoolFOption< BothFlags<[], [CC1Option], " experimental bounds safety extension for C">>; +defm lifetime_safety : BoolFOption< + "experimental-lifetime-safety", + LangOpts<"EnableLifetimeSafety">, DefaultFalse, + PosFlag<SetTrue, [], [CC1Option], "Enable">, + NegFlag<SetFalse, [], [CC1Option], "Disable">, + BothFlags<[], [CC1Option], + " experimental lifetime safety for C++">>; + defm addrsig : BoolFOption<"addrsig", CodeGenOpts<"Addrsig">, DefaultFalse, PosFlag<SetTrue, [], [ClangOption, CC1Option], "Emit">, @@ -4249,7 +4264,12 @@ def ffinite_loops: Flag<["-"], "ffinite-loops">, Group<f_Group>, def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group<f_Group>, HelpText<"Do not assume that any loop is finite.">, Visibility<[ClangOption, CC1Option]>; - +def fthinlto_distributor_EQ : Joined<["-"], "fthinlto-distributor=">, + Group<f_Group>, + HelpText<"Path to the ThinLTO distributor process. If specified, " + "ThinLTO backend compilations will be distributed by LLD">, + MetaVarName<"<path>">, + Visibility<[ClangOption, CLOption]>; def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>, HelpText<"Process trigraph sequences">, Visibility<[ClangOption, CC1Option]>; def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>, @@ -4507,6 +4527,7 @@ defm aarch64_jump_table_hardening: OptInCC1FFlag<"aarch64-jump-table-hardening", defm ptrauth_objc_isa : OptInCC1FFlag<"ptrauth-objc-isa", "Enable signing and authentication of Objective-C object's 'isa' field">; defm ptrauth_objc_interface_sel : OptInCC1FFlag<"ptrauth-objc-interface-sel", "Enable signing and authentication of Objective-C object's 'SEL' fields">; defm ptrauth_objc_class_ro : OptInCC1FFlag<"ptrauth-objc-class-ro", "Enable signing and authentication for ObjC class_ro pointers">; +defm ptrauth_block_descriptor_pointers : OptInCC1FFlag<"ptrauth-block-descriptor-pointers", "Enable signing and authentication of block descriptors">; } def fenable_matrix : Flag<["-"], "fenable-matrix">, Group<f_Group>, diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index b4f2a87fe7e8..f0d0000c42a9 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -4694,6 +4694,13 @@ struct FormatStyle { /// <conditional-body> <conditional-body> /// \endcode bool AfterIfMacros; + /// If ``true``, put a space between alternative operator ``not`` and the + /// opening parenthesis. + /// \code + /// true: false: + /// return not (a || b); vs. return not(a || b); + /// \endcode + bool AfterNot; /// If ``true``, put a space between operator overloading and opening /// parentheses. /// \code @@ -4742,9 +4749,9 @@ struct FormatStyle { : AfterControlStatements(false), AfterForeachMacros(false), AfterFunctionDeclarationName(false), AfterFunctionDefinitionName(false), AfterIfMacros(false), - AfterOverloadedOperator(false), AfterPlacementOperator(true), - AfterRequiresInClause(false), AfterRequiresInExpression(false), - BeforeNonEmptyParentheses(false) {} + AfterNot(false), AfterOverloadedOperator(false), + AfterPlacementOperator(true), AfterRequiresInClause(false), + AfterRequiresInExpression(false), BeforeNonEmptyParentheses(false) {} bool operator==(const SpaceBeforeParensCustom &Other) const { return AfterControlStatements == Other.AfterControlStatements && @@ -4753,6 +4760,7 @@ struct FormatStyle { Other.AfterFunctionDeclarationName && AfterFunctionDefinitionName == Other.AfterFunctionDefinitionName && AfterIfMacros == Other.AfterIfMacros && + AfterNot == Other.AfterNot && AfterOverloadedOperator == Other.AfterOverloadedOperator && AfterPlacementOperator == Other.AfterPlacementOperator && AfterRequiresInClause == Other.AfterRequiresInClause && diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h index 06971ff87ab9..423f2ffe2f85 100644 --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -143,9 +143,6 @@ class Lexer : public PreprocessorLexer { /// True if this is the first time we're lexing the input file. bool IsFirstTimeLexingFile; - /// True if current lexing token is the first pp-token. - bool IsFirstPPToken; - // NewLinePtr - A pointer to new line character '\n' being lexed. For '\r\n', // it also points to '\n.' const char *NewLinePtr; diff --git a/clang/include/clang/Lex/NoTrivialPPDirectiveTracer.h b/clang/include/clang/Lex/NoTrivialPPDirectiveTracer.h new file mode 100644 index 000000000000..9ab3c6a528a1 --- /dev/null +++ b/clang/include/clang/Lex/NoTrivialPPDirectiveTracer.h @@ -0,0 +1,310 @@ +//===--- NoTrivialPPDirectiveTracer.h ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the NoTrivialPPDirectiveTracer interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LEX_NO_TRIVIAL_PPDIRECTIVE_TRACER_H +#define LLVM_CLANG_LEX_NO_TRIVIAL_PPDIRECTIVE_TRACER_H + +#include "clang/Lex/PPCallbacks.h" + +namespace clang { +class Preprocessor; + +/// Consider the following code: +/// +/// # 1 __FILE__ 1 3 +/// export module a; +/// +/// According to the wording in +/// [P1857R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p1857r3.html): +/// +/// A module directive may only appear as the first preprocessing tokens in a +/// file (excluding the global module fragment.) +/// +/// and the wording in +/// [[cpp.pre]](https://eel.is/c++draft/cpp.pre#nt:module-file): +/// module-file: +/// pp-global-module-fragment[opt] pp-module group[opt] +/// pp-private-module-fragment[opt] +/// +/// `#` is the first pp-token in the translation unit, and it was rejected by +/// clang, but they really should be exempted from this rule. The goal is to not +/// allow any preprocessor conditionals or most state changes, but these don't +/// fit that. +/// +/// State change would mean most semantically observable preprocessor state, +/// particularly anything that is order dependent. Global flags like being a +/// system header/module shouldn't matter. +/// +/// We should exempt a brunch of directives, even though it violates the current +/// standard wording. +/// +/// This class used to trace 'no-trivial' pp-directives in main file, which may +/// change the preprocessing state. +/// +/// FIXME: Once the wording of the standard is revised, we need to follow the +/// wording of the standard. Currently this is just a workaround +class NoTrivialPPDirectiveTracer : public PPCallbacks { + Preprocessor &PP; + + /// Whether preprocessing main file. We only focus on the main file. + bool InMainFile = true; + + /// Whether one or more conditional, include or other 'no-trivial' + /// pp-directives has seen before. + bool SeenNoTrivialPPDirective = false; + + void setSeenNoTrivialPPDirective(); + +public: + NoTrivialPPDirectiveTracer(Preprocessor &P) : PP(P) {} + + bool hasSeenNoTrivialPPDirective() const; + + /// Callback invoked whenever the \p Lexer moves to a different file for + /// lexing. Unlike \p FileChanged line number directives and other related + /// pragmas do not trigger callbacks to \p LexedFileChanged. + /// + /// \param FID The \p FileID that the \p Lexer moved to. + /// + /// \param Reason Whether the \p Lexer entered a new file or exited one. + /// + /// \param FileType The \p CharacteristicKind of the file the \p Lexer moved + /// to. + /// + /// \param PrevFID The \p FileID the \p Lexer was using before the change. + /// + /// \param Loc The location where the \p Lexer entered a new file from or the + /// location that the \p Lexer moved into after exiting a file. + void LexedFileChanged(FileID FID, LexedFileChangeReason Reason, + SrcMgr::CharacteristicKind FileType, FileID PrevFID, + SourceLocation Loc) override; + + /// Callback invoked whenever an embed directive has been processed, + /// regardless of whether the embed will actually find a file. + /// + /// \param HashLoc The location of the '#' that starts the embed directive. + /// + /// \param FileName The name of the file being included, as written in the + /// source code. + /// + /// \param IsAngled Whether the file name was enclosed in angle brackets; + /// otherwise, it was enclosed in quotes. + /// + /// \param File The actual file that may be included by this embed directive. + /// + /// \param Params The parameters used by the directive. + void EmbedDirective(SourceLocation HashLoc, StringRef FileName, bool IsAngled, + OptionalFileEntryRef File, + const LexEmbedParametersResult &Params) override { + setSeenNoTrivialPPDirective(); + } + + /// Callback invoked whenever an inclusion directive of + /// any kind (\c \#include, \c \#import, etc.) has been processed, regardless + /// of whether the inclusion will actually result in an inclusion. + /// + /// \param HashLoc The location of the '#' that starts the inclusion + /// directive. + /// + /// \param IncludeTok The token that indicates the kind of inclusion + /// directive, e.g., 'include' or 'import'. + /// + /// \param FileName The name of the file being included, as written in the + /// source code. + /// + /// \param IsAngled Whether the file name was enclosed in angle brackets; + /// otherwise, it was enclosed in quotes. + /// + /// \param FilenameRange The character range of the quotes or angle brackets + /// for the written file name. + /// + /// \param File The actual file that may be included by this inclusion + /// directive. + /// + /// \param SearchPath Contains the search path which was used to find the file + /// in the file system. If the file was found via an absolute include path, + /// SearchPath will be empty. For framework includes, the SearchPath and + /// RelativePath will be split up. For example, if an include of "Some/Some.h" + /// is found via the framework path + /// "path/to/Frameworks/Some.framework/Headers/Some.h", SearchPath will be + /// "path/to/Frameworks/Some.framework/Headers" and RelativePath will be + /// "Some.h". + /// + /// \param RelativePath The path relative to SearchPath, at which the include + /// file was found. This is equal to FileName except for framework includes. + /// + /// \param SuggestedModule The module suggested for this header, if any. + /// + /// \param ModuleImported Whether this include was translated into import of + /// \p SuggestedModule. + /// + /// \param FileType The characteristic kind, indicates whether a file or + /// directory holds normal user code, system code, or system code which is + /// implicitly 'extern "C"' in C++ mode. + /// + void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, + StringRef FileName, bool IsAngled, + CharSourceRange FilenameRange, + OptionalFileEntryRef File, StringRef SearchPath, + StringRef RelativePath, const Module *SuggestedModule, + bool ModuleImported, + SrcMgr::CharacteristicKind FileType) override { + setSeenNoTrivialPPDirective(); + } + + /// Callback invoked whenever there was an explicit module-import + /// syntax. + /// + /// \param ImportLoc The location of import directive token. + /// + /// \param Path The identifiers (and their locations) of the module + /// "path", e.g., "std.vector" would be split into "std" and "vector". + /// + /// \param Imported The imported module; can be null if importing failed. + /// + void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path, + const Module *Imported) override { + setSeenNoTrivialPPDirective(); + } + + /// Callback invoked when the end of the main file is reached. + /// + /// No subsequent callbacks will be made. + void EndOfMainFile() override { setSeenNoTrivialPPDirective(); } + + /// Callback invoked when start reading any pragma directive. + void PragmaDirective(SourceLocation Loc, + PragmaIntroducerKind Introducer) override {} + + /// Called by Preprocessor::HandleMacroExpandedIdentifier when a + /// macro invocation is found. + void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD, + SourceRange Range, const MacroArgs *Args) override; + + /// Hook called whenever a macro definition is seen. + void MacroDefined(const Token &MacroNameTok, + const MacroDirective *MD) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever a macro \#undef is seen. + /// \param MacroNameTok The active Token + /// \param MD A MacroDefinition for the named macro. + /// \param Undef New MacroDirective if the macro was defined, null otherwise. + /// + /// MD is released immediately following this callback. + void MacroUndefined(const Token &MacroNameTok, const MacroDefinition &MD, + const MacroDirective *Undef) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever the 'defined' operator is seen. + /// \param MD The MacroDirective if the name was a macro, null otherwise. + void Defined(const Token &MacroNameTok, const MacroDefinition &MD, + SourceRange Range) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#if is seen. + /// \param Loc the source location of the directive. + /// \param ConditionRange The SourceRange of the expression being tested. + /// \param ConditionValue The evaluated value of the condition. + /// + // FIXME: better to pass in a list (or tree!) of Tokens. + void If(SourceLocation Loc, SourceRange ConditionRange, + ConditionValueKind ConditionValue) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#elif is seen. + /// \param Loc the source location of the directive. + /// \param ConditionRange The SourceRange of the expression being tested. + /// \param ConditionValue The evaluated value of the condition. + /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive. + // FIXME: better to pass in a list (or tree!) of Tokens. + void Elif(SourceLocation Loc, SourceRange ConditionRange, + ConditionValueKind ConditionValue, SourceLocation IfLoc) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#ifdef is seen. + /// \param Loc the source location of the directive. + /// \param MacroNameTok Information on the token being tested. + /// \param MD The MacroDefinition if the name was a macro, null otherwise. + void Ifdef(SourceLocation Loc, const Token &MacroNameTok, + const MacroDefinition &MD) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#elifdef branch is taken. + /// \param Loc the source location of the directive. + /// \param MacroNameTok Information on the token being tested. + /// \param MD The MacroDefinition if the name was a macro, null otherwise. + void Elifdef(SourceLocation Loc, const Token &MacroNameTok, + const MacroDefinition &MD) override { + setSeenNoTrivialPPDirective(); + } + /// Hook called whenever an \#elifdef is skipped. + /// \param Loc the source location of the directive. + /// \param ConditionRange The SourceRange of the expression being tested. + /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive. + // FIXME: better to pass in a list (or tree!) of Tokens. + void Elifdef(SourceLocation Loc, SourceRange ConditionRange, + SourceLocation IfLoc) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#ifndef is seen. + /// \param Loc the source location of the directive. + /// \param MacroNameTok Information on the token being tested. + /// \param MD The MacroDefiniton if the name was a macro, null otherwise. + void Ifndef(SourceLocation Loc, const Token &MacroNameTok, + const MacroDefinition &MD) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#elifndef branch is taken. + /// \param Loc the source location of the directive. + /// \param MacroNameTok Information on the token being tested. + /// \param MD The MacroDefinition if the name was a macro, null otherwise. + void Elifndef(SourceLocation Loc, const Token &MacroNameTok, + const MacroDefinition &MD) override { + setSeenNoTrivialPPDirective(); + } + /// Hook called whenever an \#elifndef is skipped. + /// \param Loc the source location of the directive. + /// \param ConditionRange The SourceRange of the expression being tested. + /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive. + // FIXME: better to pass in a list (or tree!) of Tokens. + void Elifndef(SourceLocation Loc, SourceRange ConditionRange, + SourceLocation IfLoc) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#else is seen. + /// \param Loc the source location of the directive. + /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive. + void Else(SourceLocation Loc, SourceLocation IfLoc) override { + setSeenNoTrivialPPDirective(); + } + + /// Hook called whenever an \#endif is seen. + /// \param Loc the source location of the directive. + /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive. + void Endif(SourceLocation Loc, SourceLocation IfLoc) override { + setSeenNoTrivialPPDirective(); + } +}; + +} // namespace clang + +#endif // LLVM_CLANG_LEX_NO_TRIVIAL_PPDIRECTIVE_TRACER_H diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 4d82e20e5d4f..e90564a9739a 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -82,6 +82,7 @@ class PreprocessorLexer; class PreprocessorOptions; class ScratchBuffer; class TargetInfo; +class NoTrivialPPDirectiveTracer; namespace Builtin { class Context; @@ -353,6 +354,11 @@ private: /// First pp-token source location in current translation unit. SourceLocation FirstPPTokenLoc; + /// A preprocessor directive tracer to trace whether the preprocessing + /// state changed. These changes would mean most semantically observable + /// preprocessor state, particularly anything that is order dependent. + NoTrivialPPDirectiveTracer *DirTracer = nullptr; + /// A position within a C++20 import-seq. class StdCXXImportSeq { public: @@ -609,6 +615,8 @@ private: return State == NamedModuleImplementation && !getName().contains(':'); } + bool isNotAModuleDecl() const { return State == NotAModuleDecl; } + StringRef getName() const { assert(isNamedModule() && "Can't get name from a non named module"); return Name; @@ -3087,6 +3095,10 @@ public: bool setDeserializedSafeBufferOptOutMap( const SmallVectorImpl<SourceLocation> &SrcLocSeqs); + /// Whether we've seen pp-directives which may have changed the preprocessing + /// state. + bool hasSeenNoTrivialPPDirective() const; + private: /// Helper functions to forward lexing to the actual lexer. They all share the /// same signature. diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h index fc43e72593b9..d9dc5a562d80 100644 --- a/clang/include/clang/Lex/Token.h +++ b/clang/include/clang/Lex/Token.h @@ -86,12 +86,12 @@ public: // macro stringizing or charizing operator. CommaAfterElided = 0x200, // The comma following this token was elided (MS). IsEditorPlaceholder = 0x400, // This identifier is a placeholder. - - IsReinjected = 0x800, // A phase 4 token that was produced before and - // re-added, e.g. via EnterTokenStream. Annotation - // tokens are *not* reinjected. - FirstPPToken = 0x1000, // This token is the first pp token in the - // translation unit. + IsReinjected = 0x800, // A phase 4 token that was produced before and + // re-added, e.g. via EnterTokenStream. Annotation + // tokens are *not* reinjected. + HasSeenNoTrivialPPDirective = + 0x1000, // Whether we've seen any 'no-trivial' pp-directives before + // current position. }; tok::TokenKind getKind() const { return Kind; } @@ -321,8 +321,9 @@ public: /// lexer uses identifier tokens to represent placeholders. bool isEditorPlaceholder() const { return getFlag(IsEditorPlaceholder); } - /// Returns true if this token is the first pp-token. - bool isFirstPPToken() const { return getFlag(FirstPPToken); } + bool hasSeenNoTrivialPPDirective() const { + return getFlag(HasSeenNoTrivialPPDirective); + } }; /// Information about the conditional stack (\#if directives) diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h index 9135ff949eea..a70335bef9dd 100644 --- a/clang/include/clang/Sema/Overload.h +++ b/clang/include/clang/Sema/Overload.h @@ -350,11 +350,6 @@ class Sema; LLVM_PREFERRED_TYPE(bool) unsigned BindsToRvalue : 1; - /// Whether this was an identity conversion with qualification - /// conversion for the implicit object argument. - LLVM_PREFERRED_TYPE(bool) - unsigned IsImplicitObjectArgumentQualificationConversion : 1; - /// Whether this binds an implicit object argument to a /// non-static member function without a ref-qualifier. LLVM_PREFERRED_TYPE(bool) @@ -453,11 +448,11 @@ class Sema; #endif return true; } + if (!C.hasSameType(getFromType(), getToType(2))) + return false; if (BindsToRvalue && IsLvalueReference) return false; - if (IsImplicitObjectArgumentQualificationConversion) - return C.hasSameUnqualifiedType(getFromType(), getToType(2)); - return C.hasSameType(getFromType(), getToType(2)); + return true; } ImplicitConversionRank getRank() const; diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index b331acbe606b..7b0be368d67f 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -9836,7 +9836,7 @@ public: SourceLocation ModuleLoc, ModuleDeclKind MDK, ModuleIdPath Path, ModuleIdPath Partition, ModuleImportState &ImportState, - bool IntroducerIsFirstPPToken); + bool SeenNoTrivialPPDirective); /// The parser has processed a global-module-fragment declaration that begins /// the definition of the global module fragment of the current module unit. diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index 0499a81cd523..862f1d5741c5 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -1725,6 +1725,9 @@ ASTContext::PointerAuthContent ASTContext::findPointerAuthContent(QualType T) { assert(isPointerAuthenticationAvailable()); T = T.getCanonicalType(); + if (T->isDependentType()) + return PointerAuthContent::None; + if (T.hasAddressDiscriminatedPointerAuth()) return PointerAuthContent::AddressDiscriminatedData; const RecordDecl *RD = T->getAsRecordDecl(); @@ -3032,7 +3035,7 @@ bool ASTContext::hasUniqueObjectRepresentations( return true; } - // All other pointers (except __ptrauth pointers) are unique. + // All other pointers are unique. if (Ty->isPointerType()) return !Ty.hasAddressDiscriminatedPointerAuth(); diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp index 3aa6b3784410..25d19a3dcd48 100644 --- a/clang/lib/AST/ASTStructuralEquivalence.cpp +++ b/clang/lib/AST/ASTStructuralEquivalence.cpp @@ -456,7 +456,9 @@ CheckStructurallyEquivalentAttributes(StructuralEquivalenceContext &Context, const Decl *D1, const Decl *D2, const Decl *PrimaryDecl = nullptr) { // If either declaration has an attribute on it, we treat the declarations - // as not being structurally equivalent. + // as not being structurally equivalent unless both declarations are implicit + // (ones generated by the compiler like __NSConstantString_tag). + // // FIXME: this should be handled on a case-by-case basis via tablegen in // Attr.td. There are multiple cases to consider: one declaration with the // attribute, another without it; different attribute syntax|spellings for @@ -468,7 +470,7 @@ CheckStructurallyEquivalentAttributes(StructuralEquivalenceContext &Context, D1Attr = *D1->getAttrs().begin(); if (D2->hasAttrs()) D2Attr = *D2->getAttrs().begin(); - if (D1Attr || D2Attr) { + if ((D1Attr || D2Attr) && !D1->isImplicit() && !D2->isImplicit()) { const auto *DiagnoseDecl = cast<TypeDecl>(PrimaryDecl ? PrimaryDecl : D2); Context.Diag2(DiagnoseDecl->getLocation(), diag::warn_odr_tag_type_with_attributes) @@ -873,7 +875,29 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, else if (T1->getTypeClass() == Type::FunctionNoProto && T2->getTypeClass() == Type::FunctionProto) TC = Type::FunctionNoProto; - else + else if (Context.LangOpts.C23 && !Context.StrictTypeSpelling && + (T1->getTypeClass() == Type::Enum || + T2->getTypeClass() == Type::Enum)) { + // In C23, if not being strict about token equivalence, we need to handle + // the case where one type is an enumeration and the other type is an + // integral type. + // + // C23 6.7.3.3p16: The enumerated type is compatible with the underlying + // type of the enumeration. + // + // Treat the enumeration as its underlying type and use the builtin type + // class comparison. + if (T1->getTypeClass() == Type::Enum) { + T1 = T1->getAs<EnumType>()->getDecl()->getIntegerType(); + if (!T2->isBuiltinType() || T1.isNull()) // Sanity check + return false; + } else if (T2->getTypeClass() == Type::Enum) { + T2 = T2->getAs<EnumType>()->getDecl()->getIntegerType(); + if (!T1->isBuiltinType() || T2.isNull()) // Sanity check + return false; + } + TC = Type::Builtin; + } else return false; } @@ -2067,6 +2091,48 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, !CheckStructurallyEquivalentAttributes(Context, D1, D2)) return false; + // In C23, if one enumeration has a fixed underlying type, the other shall + // have a compatible fixed underlying type (6.2.7). + if (Context.LangOpts.C23) { + if (D1->isFixed() != D2->isFixed()) { + if (Context.Complain) { + Context.Diag2(D2->getLocation(), + Context.getApplicableDiagnostic( + diag::err_odr_tag_type_inconsistent)) + << Context.ToCtx.getTypeDeclType(D2) + << (&Context.FromCtx != &Context.ToCtx); + Context.Diag1(D1->getLocation(), + D1->isFixed() + ? diag::note_odr_fixed_underlying_type + : diag::note_odr_missing_fixed_underlying_type) + << D1; + Context.Diag2(D2->getLocation(), + D2->isFixed() + ? diag::note_odr_fixed_underlying_type + : diag::note_odr_missing_fixed_underlying_type) + << D2; + } + return false; + } + if (D1->isFixed()) { + assert(D2->isFixed() && "enums expected to have fixed underlying types"); + if (!IsStructurallyEquivalent(Context, D1->getIntegerType(), + D2->getIntegerType())) { + if (Context.Complain) { + Context.Diag2(D2->getLocation(), + Context.getApplicableDiagnostic( + diag::err_odr_tag_type_inconsistent)) + << Context.ToCtx.getTypeDeclType(D2) + << (&Context.FromCtx != &Context.ToCtx); + Context.Diag2(D2->getLocation(), + diag::note_odr_incompatible_fixed_underlying_type) + << D2 << D2->getIntegerType() << D1->getIntegerType(); + } + return false; + } + } + } + llvm::SmallVector<const EnumConstantDecl *, 8> D1Enums, D2Enums; auto CopyEnumerators = [](auto &&Range, llvm::SmallVectorImpl<const EnumConstantDecl *> &Cont) { diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp index 451496500979..64895f4fb9d4 100644 --- a/clang/lib/AST/DeclCXX.cpp +++ b/clang/lib/AST/DeclCXX.cpp @@ -1448,6 +1448,13 @@ void CXXRecordDecl::addedMember(Decl *D) { data().StructuralIfLiteral = false; } + // If this type contains any address discriminated values we should + // have already indicated that the only special member functions that + // can possibly be trivial are the default constructor and destructor. + if (T.hasAddressDiscriminatedPointerAuth()) + data().HasTrivialSpecialMembers &= + SMF_DefaultConstructor | SMF_Destructor; + // C++14 [meta.unary.prop]p4: // T is a class type [...] with [...] no non-static data members other // than subobjects of zero size diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 419dd5dbdc69..bfecae972940 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -4441,7 +4441,8 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E, } } else if (!IsAccess) { return CompleteObject(LVal.getLValueBase(), nullptr, BaseType); - } else if (IsConstant && Info.checkingPotentialConstantExpression() && + } else if ((IsConstant || BaseType->isReferenceType()) && + Info.checkingPotentialConstantExpression() && BaseType->isLiteralType(Info.Ctx) && !VD->hasDefinition()) { // This variable might end up being constexpr. Don't diagnose it yet. } else if (IsConstant) { @@ -4478,9 +4479,11 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E, // a null BaseVal. Any constexpr-unknown variable seen here is an error: // we can't access a constexpr-unknown object. if (!BaseVal) { - Info.FFDiag(E, diag::note_constexpr_access_unknown_variable, 1) - << AK << VD; - Info.Note(VD->getLocation(), diag::note_declared_at); + if (!Info.checkingPotentialConstantExpression()) { + Info.FFDiag(E, diag::note_constexpr_access_unknown_variable, 1) + << AK << VD; + Info.Note(VD->getLocation(), diag::note_declared_at); + } return CompleteObject(); } } else if (DynamicAllocLValue DA = LVal.Base.dyn_cast<DynamicAllocLValue>()) { @@ -7906,8 +7909,9 @@ static bool checkBitCastConstexprEligibilityType(SourceLocation Loc, // so its layout is unspecified. For now, we'll simply treat these cases // as unsupported (this should only be possible with OpenCL bool vectors // whose element count isn't a multiple of the byte size). - Info->FFDiag(Loc, diag::note_constexpr_bit_cast_invalid_vector) - << QualType(VTy, 0) << EltSize << NElts << Ctx.getCharWidth(); + if (Info) + Info->FFDiag(Loc, diag::note_constexpr_bit_cast_invalid_vector) + << QualType(VTy, 0) << EltSize << NElts << Ctx.getCharWidth(); return false; } @@ -7916,8 +7920,9 @@ static bool checkBitCastConstexprEligibilityType(SourceLocation Loc, // The layout for x86_fp80 vectors seems to be handled very inconsistently // by both clang and LLVM, so for now we won't allow bit_casts involving // it in a constexpr context. - Info->FFDiag(Loc, diag::note_constexpr_bit_cast_unsupported_type) - << EltTy; + if (Info) + Info->FFDiag(Loc, diag::note_constexpr_bit_cast_unsupported_type) + << EltTy; return false; } } @@ -10929,10 +10934,6 @@ bool RecordExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E, bool ZeroInit = E->requiresZeroInitialization(); if (CheckTrivialDefaultConstructor(Info, E->getExprLoc(), FD, ZeroInit)) { - // If we've already performed zero-initialization, we're already done. - if (Result.hasValue()) - return true; - if (ZeroInit) return ZeroInitialization(E, T); @@ -14478,12 +14479,6 @@ EvaluateComparisonBinaryOperator(EvalInfo &Info, const BinaryOperator *E, if (!EvaluatePointer(E->getRHS(), RHSValue, Info) || !LHSOK) return false; - // If we have Unknown pointers we should fail if they are not global values. - if (!(IsGlobalLValue(LHSValue.getLValueBase()) && - IsGlobalLValue(RHSValue.getLValueBase())) && - (LHSValue.AllowConstexprUnknown || RHSValue.AllowConstexprUnknown)) - return false; - // Reject differing bases from the normal codepath; we special-case // comparisons to null. if (!HasSameBase(LHSValue, RHSValue)) { @@ -14545,6 +14540,10 @@ EvaluateComparisonBinaryOperator(EvalInfo &Info, const BinaryOperator *E, (LHSValue.Base && isZeroSized(RHSValue))) return DiagComparison( diag::note_constexpr_pointer_comparison_zero_sized); + if (LHSValue.AllowConstexprUnknown || RHSValue.AllowConstexprUnknown) + return DiagComparison( + diag::note_constexpr_pointer_comparison_unspecified); + // FIXME: Verify both variables are live. return Success(CmpResult::Unequal, E); } @@ -14565,7 +14564,9 @@ EvaluateComparisonBinaryOperator(EvalInfo &Info, const BinaryOperator *E, if (!LHSDesignator.Invalid && !RHSDesignator.Invalid && IsRelational) { bool WasArrayIndex; unsigned Mismatch = FindDesignatorMismatch( - getType(LHSValue.Base), LHSDesignator, RHSDesignator, WasArrayIndex); + LHSValue.Base.isNull() ? QualType() + : getType(LHSValue.Base).getNonReferenceType(), + LHSDesignator, RHSDesignator, WasArrayIndex); // At the point where the designators diverge, the comparison has a // specified value if: // - we are comparing array indices @@ -14609,7 +14610,7 @@ EvaluateComparisonBinaryOperator(EvalInfo &Info, const BinaryOperator *E, // compare pointers within the object in question; otherwise, the result // depends on where the object is located in memory. if (!LHSValue.Base.isNull() && IsRelational) { - QualType BaseTy = getType(LHSValue.Base); + QualType BaseTy = getType(LHSValue.Base).getNonReferenceType(); if (BaseTy->isIncompleteType()) return Error(E); CharUnits Size = Info.Ctx.getTypeSizeInChars(BaseTy); diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index e5a1ab2ff890..bd3dd983bca7 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -2715,6 +2715,11 @@ bool QualType::isCXX98PODType(const ASTContext &Context) const { return false; QualType CanonicalType = getTypePtr()->CanonicalType; + + // Any type that is, or contains, address discriminated data is never POD. + if (const_cast<ASTContext&>(Context).containsAddressDiscriminatedPointerAuth(CanonicalType)) + return false; + switch (CanonicalType->getTypeClass()) { // Everything not explicitly mentioned is not POD. default: @@ -2773,6 +2778,11 @@ bool QualType::isTrivialType(const ASTContext &Context) const { if (CanonicalType->isDependentType()) return false; + // Any type that is, or contains, address discriminated data is never a + // trivial type. + if (const_cast<ASTContext&>(Context).containsAddressDiscriminatedPointerAuth(CanonicalType)) + return false; + // C++0x [basic.types]p9: // Scalar types, trivial class types, arrays of such types, and // cv-qualified versions of these types are collectively called trivial @@ -2870,6 +2880,12 @@ bool QualType::isBitwiseCloneableType(const ASTContext &Context) const { if (CanonicalType->isIncompleteType()) return false; + + // Any type that is, or contains, address discriminated data is never + // bitwise clonable. + if (const_cast<ASTContext&>(Context).containsAddressDiscriminatedPointerAuth(CanonicalType)) + return false; + const auto *RD = CanonicalType->getAsRecordDecl(); // struct/union/class if (!RD) return true; @@ -3115,6 +3131,10 @@ bool QualType::isCXX11PODType(const ASTContext &Context) const { if (BaseTy->isIncompleteType()) return false; + // Any type that is, or contains, address discriminated data is non-POD. + if (const_cast<ASTContext&>(Context).containsAddressDiscriminatedPointerAuth(*this)) + return false; + // As an extension, Clang treats vector types as Scalar types. if (BaseTy->isScalarType() || BaseTy->isVectorType()) return true; diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp index 80e7c8eff671..dadb0b757a2c 100644 --- a/clang/lib/Analysis/ThreadSafety.cpp +++ b/clang/lib/Analysis/ThreadSafety.cpp @@ -1331,7 +1331,7 @@ void ThreadSafetyAnalyzer::addLock(FactSet &FSet, FSet.removeLock(FactMan, NegC); } else { - if (inCurrentScope(*Entry) && !Entry->asserted()) + if (inCurrentScope(*Entry) && !Entry->asserted() && !Entry->reentrant()) Handler.handleNegativeNotHeld(Entry->getKind(), Entry->toString(), NegC.toString(), Entry->loc()); } diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp index af1111a86330..5d11578893c6 100644 --- a/clang/lib/Basic/Targets.cpp +++ b/clang/lib/Basic/Targets.cpp @@ -769,6 +769,9 @@ std::unique_ptr<TargetInfo> AllocateTarget(const llvm::Triple &Triple, case llvm::Triple::FreeBSD: return std::make_unique<FreeBSDTargetInfo<LoongArch64TargetInfo>>(Triple, Opts); + case llvm::Triple::OpenBSD: + return std::make_unique<OpenBSDTargetInfo<LoongArch64TargetInfo>>(Triple, + Opts); default: return std::make_unique<LoongArch64TargetInfo>(Triple, Opts); } diff --git a/clang/lib/Basic/Targets/LoongArch.cpp b/clang/lib/Basic/Targets/LoongArch.cpp index f6915df1520b..8e29bb745734 100644 --- a/clang/lib/Basic/Targets/LoongArch.cpp +++ b/clang/lib/Basic/Targets/LoongArch.cpp @@ -461,6 +461,8 @@ LoongArchTargetInfo::parseTargetAttr(StringRef Features) const { case AttrFeatureKind::Feature: Ret.Features.push_back("+" + Value.str()); + if (Value == "lasx") + Ret.Features.push_back("+lsx"); break; } } diff --git a/clang/lib/Basic/Targets/Mips.h b/clang/lib/Basic/Targets/Mips.h index 35501ed44ccd..e199df32f56e 100644 --- a/clang/lib/Basic/Targets/Mips.h +++ b/clang/lib/Basic/Targets/Mips.h @@ -129,7 +129,7 @@ public: LongWidth = LongAlign = 32; MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 32; PointerWidth = PointerAlign = 32; - PtrDiffType = SignedInt; + PtrDiffType = IntPtrType = SignedInt; SizeType = UnsignedInt; SuitableAlign = 64; } @@ -155,7 +155,7 @@ public: IntMaxType = Int64Type; LongWidth = LongAlign = 64; PointerWidth = PointerAlign = 64; - PtrDiffType = SignedLong; + PtrDiffType = IntPtrType = SignedLong; SizeType = UnsignedLong; } @@ -165,7 +165,7 @@ public: IntMaxType = Int64Type; LongWidth = LongAlign = 32; PointerWidth = PointerAlign = 32; - PtrDiffType = SignedInt; + PtrDiffType = IntPtrType = SignedInt; SizeType = UnsignedInt; } diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h index 30d861a7ca60..c1a68f464e83 100644 --- a/clang/lib/Basic/Targets/OSTargets.h +++ b/clang/lib/Basic/Targets/OSTargets.h @@ -174,6 +174,9 @@ protected: DefineStd(Builder, "unix", Opts); if (this->HasFloat128) Builder.defineMacro("__FLOAT128__"); + + if (Opts.C11) + Builder.defineMacro("__STDC_NO_THREADS__"); } public: @@ -496,6 +499,7 @@ public: case llvm::Triple::sparcv9: this->MCountName = "_mcount"; break; + case llvm::Triple::loongarch64: case llvm::Triple::riscv64: break; } diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h index ebc59c92f4c2..a7be080695ed 100644 --- a/clang/lib/Basic/Targets/X86.h +++ b/clang/lib/Basic/Targets/X86.h @@ -649,6 +649,7 @@ public: : X86_32TargetInfo(Triple, Opts) { this->WCharType = TargetInfo::UnsignedShort; this->WIntType = TargetInfo::UnsignedInt; + this->UseMicrosoftManglingForC = true; DoubleAlign = LongLongAlign = 64; resetDataLayout("e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-" "i128:128-f80:32-n8:16:32-a:0:32-S32", @@ -986,6 +987,7 @@ public: : X86_64TargetInfo(Triple, Opts) { this->WCharType = TargetInfo::UnsignedShort; this->WIntType = TargetInfo::UnsignedInt; + this->UseMicrosoftManglingForC = true; } void getTargetDefines(const LangOptions &Opts, diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h index a748ddaa110a..4e7f3561ac04 100644 --- a/clang/lib/CodeGen/Address.h +++ b/clang/lib/CodeGen/Address.h @@ -176,6 +176,11 @@ public: static Address invalid() { return Address(nullptr); } bool isValid() const { return Pointer.getPointer() != nullptr; } + llvm::Value *getPointerIfNotSigned() const { + assert(isValid() && "pointer isn't valid"); + return !isSigned() ? Pointer.getPointer() : nullptr; + } + /// This function is used in situations where the caller is doing some sort of /// opaque "laundering" of the pointer. void replaceBasePointer(llvm::Value *P) { diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp index 1aba841eb5fc..c1f5b983f723 100644 --- a/clang/lib/CodeGen/CGBlocks.cpp +++ b/clang/lib/CodeGen/CGBlocks.cpp @@ -188,13 +188,14 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM, // Optional copy/dispose helpers. bool hasInternalHelper = false; if (blockInfo.NeedsCopyDispose) { + auto &Schema = CGM.getCodeGenOpts().PointerAuth.BlockHelperFunctionPointers; // copy_func_helper_decl llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo); - elements.add(copyHelper); + elements.addSignedPointer(copyHelper, Schema, GlobalDecl(), QualType()); // destroy_func_decl llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo); - elements.add(disposeHelper); + elements.addSignedPointer(disposeHelper, Schema, GlobalDecl(), QualType()); if (cast<llvm::Function>(copyHelper->stripPointerCasts()) ->hasInternalLinkage() || @@ -567,9 +568,8 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true); info.CanBeGlobal = true; return; - } - else if (C.getLangOpts().ObjC && - CGM.getLangOpts().getGC() == LangOptions::NonGC) + } else if (C.getLangOpts().ObjC && + CGM.getLangOpts().getGC() == LangOptions::NonGC) info.HasCapturedVariableLayout = true; if (block->doesNotEscape()) @@ -783,7 +783,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) { llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL; - auto GenVoidPtrTy = + llvm::PointerType *GenVoidPtrTy = IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy; LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default; auto GenVoidPtrSize = CharUnits::fromQuantity( @@ -817,9 +817,6 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { : CGM.getNSConcreteStackBlock(); isa = blockISA; - // Build the block descriptor. - descriptor = buildBlockDescriptor(CGM, blockInfo); - // Compute the initial on-stack block flags. if (!CGM.getCodeGenOpts().DisableBlockSignatureString) flags = BLOCK_HAS_SIGNATURE; @@ -833,6 +830,9 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { flags |= BLOCK_USE_STRET; if (blockInfo.NoEscape) flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL; + + // Build the block descriptor. + descriptor = buildBlockDescriptor(CGM, blockInfo); } auto projectField = [&](unsigned index, const Twine &name) -> Address { @@ -883,11 +883,25 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { llvm::ConstantInt::get(IntTy, blockInfo.BlockAlign.getQuantity()), getIntSize(), "block.align"); } - addHeaderField(blockFn, GenVoidPtrSize, "block.invoke"); - if (!IsOpenCL) - addHeaderField(descriptor, getPointerSize(), "block.descriptor"); - else if (auto *Helper = - CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) { + + if (!IsOpenCL) { + llvm::Value *blockFnPtr = + llvm::ConstantExpr::getBitCast(InvokeFn, VoidPtrTy); + QualType type = blockInfo.getBlockExpr() + ->getType() + ->castAs<BlockPointerType>() + ->getPointeeType(); + addSignedHeaderField( + blockFnPtr, + CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers, + GlobalDecl(), type, getPointerSize(), "block.invoke"); + + addSignedHeaderField( + descriptor, CGM.getCodeGenOpts().PointerAuth.BlockDescriptorPointers, + GlobalDecl(), type, getPointerSize(), "block.descriptor"); + } else if (auto *Helper = + CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) { + addHeaderField(blockFn, GenVoidPtrSize, "block.invoke"); for (auto I : Helper->getCustomFieldValues(*this, blockInfo)) { addHeaderField( I.first, @@ -895,7 +909,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { CGM.getDataLayout().getTypeAllocSize(I.first->getType())), I.second); } - } + } else + addHeaderField(blockFn, GenVoidPtrSize, "block.invoke"); } // Finally, capture all the values into the block. @@ -1166,6 +1181,8 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, ASTContext &Ctx = getContext(); CallArgList Args; + llvm::Value *FuncPtr = nullptr; + if (getLangOpts().OpenCL) { // For OpenCL, BlockPtr is already casted to generic block literal. @@ -1185,7 +1202,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, if (!isa<ParmVarDecl>(E->getCalleeDecl())) Func = CGM.getOpenCLRuntime().getInvokeFunction(E->getCallee()); else { - llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 2); + FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 2); Func = Builder.CreateAlignedLoad(GenericVoidPtrTy, FuncPtr, getPointerAlign()); } @@ -1194,7 +1211,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, BlockPtr = Builder.CreatePointerCast(BlockPtr, UnqualPtrTy, "block.literal"); // Get pointer to the block invoke function - llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 3); + FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 3); // First argument is a block literal casted to a void pointer BlockPtr = Builder.CreatePointerCast(BlockPtr, VoidPtrTy); @@ -1211,7 +1228,15 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, CGM.getTypes().arrangeBlockFunctionCall(Args, FuncTy); // Prepare the callee. - CGCallee Callee(CGCalleeInfo(), Func); + CGPointerAuthInfo PointerAuth; + if (auto &AuthSchema = + CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers) { + assert(FuncPtr != nullptr && "Missing function pointer for AuthInfo"); + PointerAuth = + EmitPointerAuthInfo(AuthSchema, FuncPtr, GlobalDecl(), FnType); + } + + CGCallee Callee(CGCalleeInfo(), Func, PointerAuth); // And call the block. return EmitCall(FnInfo, Callee, ReturnValue, Args, CallOrInvoke); @@ -1295,14 +1320,15 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, bool IsOpenCL = CGM.getLangOpts().OpenCL; bool IsWindows = CGM.getTarget().getTriple().isOSWindows(); + auto &CGOPointerAuth = CGM.getCodeGenOpts().PointerAuth; if (!IsOpenCL) { // isa if (IsWindows) fields.addNullPointer(CGM.Int8PtrPtrTy); else fields.addSignedPointer(CGM.getNSConcreteGlobalBlock(), - CGM.getCodeGenOpts().PointerAuth.ObjCIsaPointers, - GlobalDecl(), QualType()); + CGOPointerAuth.ObjCIsaPointers, GlobalDecl(), + QualType()); // __flags BlockFlags flags = BLOCK_IS_GLOBAL; @@ -1321,11 +1347,20 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, } // Function - fields.add(blockFn); + if (auto &Schema = CGOPointerAuth.BlockInvocationFunctionPointers) { + QualType FnType = blockInfo.getBlockExpr() + ->getType() + ->castAs<BlockPointerType>() + ->getPointeeType(); + fields.addSignedPointer(blockFn, Schema, GlobalDecl(), FnType); + } else + fields.add(blockFn); if (!IsOpenCL) { // Descriptor - fields.add(buildBlockDescriptor(CGM, blockInfo)); + llvm::Constant *Descriptor = buildBlockDescriptor(CGM, blockInfo); + fields.addSignedPointer(Descriptor, CGOPointerAuth.BlockDescriptorPointers, + GlobalDecl(), QualType()); } else if (auto *Helper = CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) { for (auto *I : Helper->getCustomFieldValues(CGM, blockInfo)) { @@ -1995,8 +2030,8 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { // it. It's not quite worth the annoyance to avoid creating it in the // first place. if (!needsEHCleanup(captureType.isDestructedType())) - if (auto *I = - cast_or_null<llvm::Instruction>(dstField.getBasePointer())) + if (auto *I = cast_or_null<llvm::Instruction>( + dstField.getPointerIfNotSigned())) I->eraseFromParent(); } break; @@ -2730,8 +2765,16 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { unsigned nextHeaderIndex = 0; CharUnits nextHeaderOffset; auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize, - const Twine &name) { + const Twine &name, bool isFunction = false) { auto fieldAddr = Builder.CreateStructGEP(addr, nextHeaderIndex, name); + if (isFunction) { + if (auto &Schema = CGM.getCodeGenOpts() + .PointerAuth.BlockByrefHelperFunctionPointers) { + auto PointerAuth = EmitPointerAuthInfo( + Schema, fieldAddr.emitRawPointer(*this), GlobalDecl(), QualType()); + value = EmitPointerAuthSign(PointerAuth, value); + } + } Builder.CreateStore(value, fieldAddr); nextHeaderIndex++; @@ -2814,10 +2857,10 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { storeHeaderField(V, getIntSize(), "byref.size"); if (helpers) { - storeHeaderField(helpers->CopyHelper, getPointerSize(), - "byref.copyHelper"); + storeHeaderField(helpers->CopyHelper, getPointerSize(), "byref.copyHelper", + /*isFunction=*/true); storeHeaderField(helpers->DisposeHelper, getPointerSize(), - "byref.disposeHelper"); + "byref.disposeHelper", /*isFunction=*/true); } if (ByRefHasLifetime && HasByrefExtendedLayout) { diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index c8c3d6b20c49..5344a9710d64 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -4787,19 +4787,6 @@ struct DestroyUnpassedArg final : EHScopeStack::Cleanup { } }; -struct DisableDebugLocationUpdates { - CodeGenFunction &CGF; - bool disabledDebugInfo; - DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) { - if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo())) - CGF.disableDebugInfo(); - } - ~DisableDebugLocationUpdates() { - if (disabledDebugInfo) - CGF.enableDebugInfo(); - } -}; - } // end anonymous namespace RValue CallArg::getRValue(CodeGenFunction &CGF) const { @@ -4836,7 +4823,9 @@ void CodeGenFunction::EmitWritebacks(const CallArgList &args) { void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, QualType type) { - DisableDebugLocationUpdates Dis(*this, E); + std::optional<DisableDebugLocationUpdates> Dis; + if (isa<CXXDefaultArgExpr>(E)) + Dis.emplace(*this); if (const ObjCIndirectCopyRestoreExpr *CRE = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) { assert(getLangOpts().ObjCAutoRefCount); @@ -6229,3 +6218,12 @@ RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr, return CGM.getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty, Slot); return CGM.getABIInfo().EmitVAArg(*this, VAListAddr, Ty, Slot); } + +DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF) + : CGF(CGF) { + CGF.disableDebugInfo(); +} + +DisableDebugLocationUpdates::~DisableDebugLocationUpdates() { + CGF.enableDebugInfo(); +} diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index 0b4e3f9cb036..17b2ce558f71 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -457,6 +457,12 @@ inline FnInfoOpts &operator&=(FnInfoOpts &A, FnInfoOpts B) { return A; } +struct DisableDebugLocationUpdates { + CodeGenFunction &CGF; + DisableDebugLocationUpdates(CodeGenFunction &CGF); + ~DisableDebugLocationUpdates(); +}; + } // end namespace CodeGen } // end namespace clang diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index 0fc488e98aaf..117ef3d16e21 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -707,11 +707,15 @@ struct GetReturnObjectManager { Builder.CreateStore(Builder.getFalse(), GroActiveFlag); GroEmission = CGF.EmitAutoVarAlloca(*GroVarDecl); - auto *GroAlloca = dyn_cast_or_null<llvm::AllocaInst>( - GroEmission.getOriginalAllocatedAddress().getPointer()); - assert(GroAlloca && "expected alloca to be emitted"); - GroAlloca->setMetadata(llvm::LLVMContext::MD_coro_outside_frame, - llvm::MDNode::get(CGF.CGM.getLLVMContext(), {})); + + if (!GroVarDecl->isNRVOVariable()) { + // NRVO variables don't have allocas and won't have the same issue. + auto *GroAlloca = dyn_cast_or_null<llvm::AllocaInst>( + GroEmission.getOriginalAllocatedAddress().getPointer()); + assert(GroAlloca && "expected alloca to be emitted"); + GroAlloca->setMetadata(llvm::LLVMContext::MD_coro_outside_frame, + llvm::MDNode::get(CGF.CGM.getLLVMContext(), {})); + } // Remember the top of EHStack before emitting the cleanup. auto old_top = CGF.EHStack.stable_begin(); diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index f97c7b644598..ec28bd259e8e 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -170,6 +170,10 @@ void CGDebugInfo::addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction, if (!Group || !CGM.getCodeGenOpts().DebugKeyInstructions) return; + llvm::DISubprogram *SP = KeyInstruction->getFunction()->getSubprogram(); + if (!SP || !SP->getKeyInstructionsEnabled()) + return; + addInstSourceAtomMetadata(KeyInstruction, Group, /*Rank=*/1); llvm::Instruction *BackupI = @@ -2641,7 +2645,8 @@ StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) { // existing information in the DWARF. The type is assumed to be 'void *'. void CGDebugInfo::emitVTableSymbol(llvm::GlobalVariable *VTable, const CXXRecordDecl *RD) { - if (!CGM.getTarget().getCXXABI().isItaniumFamily()) + if (!CGM.getTarget().getCXXABI().isItaniumFamily() || + CGM.getTarget().getTriple().isOSBinFormatCOFF()) return; ASTContext &Context = CGM.getContext(); diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 85c768807572..b5debd93b0f6 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3314,7 +3314,14 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { auto *FD = LambdaCaptureFields.lookup(BD); return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue); } - return EmitLValue(BD->getBinding()); + // Suppress debug location updates when visiting the binding, since the + // binding may emit instructions that would otherwise be associated with the + // binding itself, rather than the expression referencing the binding. (this + // leads to jumpy debug stepping behavior where the location/debugger jump + // back to the binding declaration, then back to the expression referencing + // the binding) + DisableDebugLocationUpdates D(*this); + return EmitLValue(BD->getBinding(), NotKnownNonNull); } // We can form DeclRefExprs naming GUID declarations when reconstituting diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 359e30cb8f5c..912b1d72c7e2 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -2313,7 +2313,8 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, bool IsExact = !IsDynamicCastToVoid && CGM.getCodeGenOpts().OptimizationLevel > 0 && DestRecordTy->getAsCXXRecordDecl()->isEffectivelyFinal() && - CGM.getCXXABI().shouldEmitExactDynamicCast(DestRecordTy); + CGM.getCXXABI().shouldEmitExactDynamicCast(DestRecordTy) && + !getLangOpts().PointerAuthCalls; // C++ [expr.dynamic.cast]p4: // If the value of v is a null pointer value in the pointer case, the result diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index 715bd392f59f..0cc468ca3ab7 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -873,8 +873,9 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, for (const BaseInfo &Base : Bases) { bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl; - Build(Val.getStructBase(Base.Index), Base.Decl, IsPrimaryBase, - VTableClass, Offset + Base.Offset); + if (!Build(Val.getStructBase(Base.Index), Base.Decl, IsPrimaryBase, + VTableClass, Offset + Base.Offset)) + return false; } } @@ -1620,7 +1621,7 @@ llvm::Constant *ConstantEmitter::tryEmitConstantExpr(const ConstantExpr *CE) { if (CE->isGLValue()) RetType = CGM.getContext().getLValueReferenceType(RetType); - return emitAbstract(CE->getBeginLoc(), CE->getAPValueResult(), RetType); + return tryEmitAbstract(CE->getAPValueResult(), RetType); } llvm::Constant * diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index e0650067b954..1a8c6f015bda 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -846,11 +846,13 @@ void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) { if (HaveInsertPoint()) EmitStopPoint(&S); + ApplyAtomGroup Grp(getDebugInfo()); EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel())); } void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) { + ApplyAtomGroup Grp(getDebugInfo()); if (const LabelDecl *Target = S.getConstantTarget()) { EmitBranchThroughCleanup(getJumpDestForLabel(Target)); return; @@ -869,6 +871,8 @@ void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) { cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB); EmitBranch(IndGotoBB); + if (CurBB && CurBB->getTerminator()) + addInstToCurrentSourceAtom(CurBB->getTerminator(), nullptr); } void CodeGenFunction::EmitIfStmt(const IfStmt &S) { @@ -2672,6 +2676,9 @@ static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect, llvm::ConstantAsMetadata::get(Loc))); } + // Make inline-asm calls Key for the debug info feature Key Instructions. + CGF.addInstToNewSourceAtom(&Result, nullptr); + if (!NoConvergent && CGF.getLangOpts().assumeFunctionsAreConvergent()) // Conservatively, mark all inline asm blocks in CUDA or OpenCL as // convergent (meaning, they may call an intrinsically convergent op, such @@ -2750,6 +2757,7 @@ EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S, } } + ApplyAtomGroup Grp(CGF.getDebugInfo()); LValue Dest = ResultRegDests[i]; // ResultTypeRequiresCast elements correspond to the first // ResultTypeRequiresCast.size() elements of RegResults. @@ -2757,7 +2765,8 @@ EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S, unsigned Size = CGF.getContext().getTypeSize(ResultRegQualTys[i]); Address A = Dest.getAddress().withElementType(ResultRegTypes[i]); if (CGF.getTargetHooks().isScalarizableAsmOperand(CGF, TruncTy)) { - Builder.CreateStore(Tmp, A); + llvm::StoreInst *S = Builder.CreateStore(Tmp, A); + CGF.addInstToCurrentSourceAtom(S, S->getValueOperand()); continue; } diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 6c32c98cec01..f5ac9f387c64 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -727,7 +727,7 @@ public: }; /// Header for data within LifetimeExtendedCleanupStack. - struct LifetimeExtendedCleanupHeader { + struct alignas(uint64_t) LifetimeExtendedCleanupHeader { /// The size of the following cleanup object. unsigned Size; /// The kind of cleanup to push. @@ -949,7 +949,8 @@ public: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size + (Header.IsConditional ? sizeof(ActiveFlag) : 0)); - static_assert(sizeof(Header) % alignof(T) == 0, + static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) && + (alignof(T) == alignof(RawAddress)), "Cleanup will be allocated on misaligned address"); char *Buffer = &LifetimeExtendedCleanupStack[OldSize]; new (Buffer) LifetimeExtendedCleanupHeader(Header); diff --git a/clang/lib/CodeGen/EHScopeStack.h b/clang/lib/CodeGen/EHScopeStack.h index ed11dc2bb05d..54f6ceaa52b9 100644 --- a/clang/lib/CodeGen/EHScopeStack.h +++ b/clang/lib/CodeGen/EHScopeStack.h @@ -143,7 +143,7 @@ public: /// /// Cleanup implementations should generally be declared in an /// anonymous namespace. - class Cleanup { + class alignas(uint64_t) Cleanup { // Anchor the construction vtable. virtual void anchor(); diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp index 89e3f6f203df..99194cd7fbb3 100644 --- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp @@ -20,6 +20,951 @@ using namespace clang; using namespace CodeGen; using namespace llvm; +// The 0th bit simulates the `vta` of RVV +// The 1st bit simulates the `vma` of RVV +static constexpr unsigned RVV_VTA = 0x1; +static constexpr unsigned RVV_VMA = 0x2; + +// RISC-V Vector builtin helper functions are marked NOINLINE to prevent +// excessive inlining in CodeGenFunction::EmitRISCVBuiltinExpr's large switch +// statement, which would significantly increase compilation time. +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVLEFFBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + if (IsMasked) { + // Move mask to right before vl. + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + IntrinsicTypes = {ResultType, Ops[4]->getType(), Ops[2]->getType()}; + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + IntrinsicTypes = {ResultType, Ops[3]->getType(), Ops[1]->getType()}; + } + Value *NewVL = Ops[2]; + Ops.erase(Ops.begin() + 2); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); + // Store new_vl. + clang::CharUnits Align; + if (IsMasked) + Align = CGM.getNaturalPointeeTypeAlignment( + E->getArg(E->getNumArgs() - 2)->getType()); + else + Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); + llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); + Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); + return V; +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVSSEBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + if (IsMasked) { + // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, + // mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVIndexedStoreBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + if (IsMasked) { + // Builtin: (mask, ptr, index, value, vl). + // Intrinsic: (value, ptr, index, mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, index, value, vl). + // Intrinsic: (value, ptr, index, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoUnaryBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoVNotBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getAllOnesValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, po2, mask, vl, policy + IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoMaskBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // op1, vl + IntrinsicTypes = {ResultType, Ops[1]->getType()}; + Ops.insert(Ops.begin() + 1, Ops[0]); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVPseudoVFUnaryBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.insert(Ops.begin() + 2, Ops[1]); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl + IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + // op1, po2, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType()}; + Ops.insert(Ops.begin() + 2, Ops[1]); + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoVWCVTBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(Ops[1]->getType())->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, Ops[1]->getType(), ElemTy, Ops[4]->getType()}; + } else { + // passtru, op1, op2, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), ElemTy, Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoVNCVTBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + Ops.insert(Ops.begin() + 2, + llvm::Constant::getNullValue(Ops.back()->getType())); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, xlen, mask, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[4]->getType(), + Ops[4]->getType()}; + } else { + // passthru, op1, xlen, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType(), + Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVlenbBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + LLVMContext &Context = CGM.getLLVMContext(); + llvm::MDBuilder MDHelper(Context); + llvm::Metadata *OpsMD[] = {llvm::MDString::get(Context, "vlenb")}; + llvm::MDNode *RegName = llvm::MDNode::get(Context, OpsMD); + llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); + llvm::Function *F = + CGM.getIntrinsic(llvm::Intrinsic::read_register, {CGF->SizeTy}); + return Builder.CreateCall(F, Metadata); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVsetvliBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::Function *F = CGM.getIntrinsic(ID, {ResultType}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVSEMaskBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + if (IsMasked) { + // Builtin: (mask, ptr, value, vl). + // Intrinsic: (value, ptr, mask, vl) + std::swap(Ops[0], Ops[2]); + } else { + // Builtin: (ptr, value, vl). + // Intrinsic: (value, ptr, vl) + std::swap(Ops[0], Ops[1]); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVUnitStridedSegLoadTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + if (IsMasked) + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[0]->getType(), + Ops.back()->getType()}; + else + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), + Ops.back()->getType()}; + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (NoPassthru) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + if (ReturnValue.isNull()) + return LoadValue; + return Builder.CreateStore(LoadValue, ReturnValue.getValue()); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVUnitStridedSegStoreTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + // Masked + // Builtin: (mask, ptr, v_tuple, vl) + // Intrinsic: (tuple, ptr, mask, vl, SegInstSEW) + // Unmasked + // Builtin: (ptr, v_tuple, vl) + // Intrinsic: (tuple, ptr, vl, SegInstSEW) + if (IsMasked) + std::swap(Ops[0], Ops[2]); + else + std::swap(Ops[0], Ops[1]); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVUnitStridedSegLoadFFTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + if (IsMasked) + IntrinsicTypes = {ResultType, Ops.back()->getType(), Ops[Offset]->getType(), + Ops[0]->getType()}; + else + IntrinsicTypes = {ResultType, Ops.back()->getType(), + Ops[Offset]->getType()}; + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (NoPassthru) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + Value *NewVL = Ops[2]; + Ops.erase(Ops.begin() + 2); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + // Get alignment from the new vl operand + clang::CharUnits Align = + CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType()); + llvm::Value *ReturnTuple = Builder.CreateExtractValue(LoadValue, 0); + // Store new_vl + llvm::Value *V = Builder.CreateExtractValue(LoadValue, 1); + Builder.CreateStore(V, Address(NewVL, V->getType(), Align)); + if (ReturnValue.isNull()) + return ReturnTuple; + return Builder.CreateStore(ReturnTuple, ReturnValue.getValue()); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVStridedSegLoadTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + if (IsMasked) + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType(), + Ops[0]->getType()}; + else + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), + Ops.back()->getType()}; + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (NoPassthru) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + if (ReturnValue.isNull()) + return LoadValue; + return Builder.CreateStore(LoadValue, ReturnValue.getValue()); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVStridedSegStoreTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 4> IntrinsicTypes; + // Masked + // Builtin: (mask, ptr, stride, v_tuple, vl) + // Intrinsic: (tuple, ptr, stride, mask, vl, SegInstSEW) + // Unmasked + // Builtin: (ptr, stride, v_tuple, vl) + // Intrinsic: (tuple, ptr, stride, vl, SegInstSEW) + if (IsMasked) + std::swap(Ops[0], Ops[3]); + else + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[4]->getType(), + Ops[3]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVAveragingBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, + // policy) + + bool HasMaskedOff = + !((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + if (!HasMaskedOff) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = CGM.getIntrinsic( + ID, {ResultType, Ops[2]->getType(), Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVNarrowingClipBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, + // policy) + + bool HasMaskedOff = + !((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + if (!HasMaskedOff) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = + CGM.getIntrinsic(ID, {ResultType, Ops[1]->getType(), Ops[2]->getType(), + Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVFloatingPointBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + bool HasMaskedOff = + !((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = + IsMasked ? (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) + : (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); + + if (!HasRoundModeOp) + Ops.insert(Ops.end() - 1, + ConstantInt::get(Ops.back()->getType(), 7)); // frm + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + if (!HasMaskedOff) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = CGM.getIntrinsic( + ID, {ResultType, Ops[2]->getType(), Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVWideningFloatingPointBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + bool HasMaskedOff = + !((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = + IsMasked ? (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) + : (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); + + if (!HasRoundModeOp) + Ops.insert(Ops.end() - 1, + ConstantInt::get(Ops.back()->getType(), 7)); // frm + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + if (!HasMaskedOff) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = + CGM.getIntrinsic(ID, {ResultType, Ops[1]->getType(), Ops[2]->getType(), + Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVIndexedSegLoadTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 5> IntrinsicTypes; + + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (NoPassthru) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + + if (IsMasked) + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + + if (ReturnValue.isNull()) + return LoadValue; + return Builder.CreateStore(LoadValue, ReturnValue.getValue()); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVIndexedSegStoreTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 5> IntrinsicTypes; + // Masked + // Builtin: (mask, ptr, index, v_tuple, vl) + // Intrinsic: (tuple, ptr, index, mask, vl, SegInstSEW) + // Unmasked + // Builtin: (ptr, index, v_tuple, vl) + // Intrinsic: (tuple, ptr, index, vl, SegInstSEW) + + if (IsMasked) + std::swap(Ops[0], Ops[3]); + else + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVFMABuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (vector_in, vector_in/scalar_in, vector_in, round_mode, + // vl, policy) + // Masked: (vector_in, vector_in/scalar_in, vector_in, mask, frm, + // vl, policy) + + bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; + + if (!HasRoundModeOp) + Ops.insert(Ops.end() - 1, + ConstantInt::get(Ops.back()->getType(), 7)); // frm + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = CGM.getIntrinsic( + ID, {ResultType, Ops[1]->getType(), Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVWideningFMABuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (vector_in, vector_in/scalar_in, vector_in, round_mode, vl, + // policy) Masked: (vector_in, vector_in/scalar_in, vector_in, mask, frm, + // vl, policy) + + bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; + + if (!HasRoundModeOp) + Ops.insert(Ops.end() - 1, + ConstantInt::get(Ops.back()->getType(), 7)); // frm + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 4); + + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = + CGM.getIntrinsic(ID, {ResultType, Ops[1]->getType(), Ops[2]->getType(), + Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVFloatingUnaryBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (passthru, op0, round_mode, vl) + // Masked: (passthru, op0, mask, frm, vl, policy) + + bool HasMaskedOff = + !((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = + IsMasked ? (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) + : (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); + + if (!HasRoundModeOp) + Ops.insert(Ops.end() - 1, + ConstantInt::get(Ops.back()->getType(), 7)); // frm + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + if (!HasMaskedOff) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVFloatingConvBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (passthru, op0, frm, vl) + // Masked: (passthru, op0, mask, frm, vl, policy) + bool HasMaskedOff = + !((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = + IsMasked ? (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) + : (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); + + if (!HasRoundModeOp) + Ops.insert(Ops.end() - 1, + ConstantInt::get(Ops.back()->getType(), 7)); // frm + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + if (!HasMaskedOff) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = CGM.getIntrinsic( + ID, {ResultType, Ops[1]->getType(), Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVFloatingReductionBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 3> IntrinsicTypes; + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + bool HasMaskedOff = + !((IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = + IsMasked ? (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) + : (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); + + if (!HasRoundModeOp) + Ops.insert(Ops.end() - 1, + ConstantInt::get(Ops.back()->getType(), 7)); // frm + + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + + if (!HasMaskedOff) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + + llvm::Function *F = CGM.getIntrinsic( + ID, {ResultType, Ops[1]->getType(), Ops.back()->getType()}); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVReinterpretBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + + if (ResultType->isIntOrIntVectorTy(1) || + Ops[0]->getType()->isIntOrIntVectorTy(1)) { + assert(isa<ScalableVectorType>(ResultType) && + isa<ScalableVectorType>(Ops[0]->getType())); + + LLVMContext &Context = CGM.getLLVMContext(); + ScalableVectorType *Boolean64Ty = + ScalableVectorType::get(llvm::Type::getInt1Ty(Context), 64); + + if (ResultType->isIntOrIntVectorTy(1)) { + // Casting from m1 vector integer -> vector boolean + // Ex: <vscale x 8 x i8> + // --(bitcast)--------> <vscale x 64 x i1> + // --(vector_extract)-> <vscale x 8 x i1> + llvm::Value *BitCast = Builder.CreateBitCast(Ops[0], Boolean64Ty); + return Builder.CreateExtractVector(ResultType, BitCast, + ConstantInt::get(CGF->Int64Ty, 0)); + } else { + // Casting from vector boolean -> m1 vector integer + // Ex: <vscale x 1 x i1> + // --(vector_insert)-> <vscale x 64 x i1> + // --(bitcast)-------> <vscale x 8 x i8> + llvm::Value *Boolean64Val = Builder.CreateInsertVector( + Boolean64Ty, llvm::PoisonValue::get(Boolean64Ty), Ops[0], + ConstantInt::get(CGF->Int64Ty, 0)); + return Builder.CreateBitCast(Boolean64Val, ResultType); + } + } + return Builder.CreateBitCast(Ops[0], ResultType); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVGetBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto *VecTy = cast<ScalableVectorType>(ResultType); + if (auto *OpVecTy = dyn_cast<ScalableVectorType>(Ops[0]->getType())) { + unsigned MaxIndex = + OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); + assert(isPowerOf2_32(MaxIndex)); + // Mask to only valid indices. + Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); + Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); + Ops[1] = + Builder.CreateMul(Ops[1], ConstantInt::get(Ops[1]->getType(), + VecTy->getMinNumElements())); + return Builder.CreateExtractVector(ResultType, Ops[0], Ops[1]); + } + + return Builder.CreateIntrinsic( + Intrinsic::riscv_tuple_extract, {ResultType, Ops[0]->getType()}, + {Ops[0], Builder.CreateTrunc(Ops[1], Builder.getInt32Ty())}); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVSetBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + if (auto *ResVecTy = dyn_cast<ScalableVectorType>(ResultType)) { + auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); + unsigned MaxIndex = + ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); + assert(isPowerOf2_32(MaxIndex)); + // Mask to only valid indices. + Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); + Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); + Ops[1] = + Builder.CreateMul(Ops[1], ConstantInt::get(Ops[1]->getType(), + VecTy->getMinNumElements())); + return Builder.CreateInsertVector(ResultType, Ops[0], Ops[2], Ops[1]); + } + + return Builder.CreateIntrinsic( + Intrinsic::riscv_tuple_insert, {ResultType, Ops[2]->getType()}, + {Ops[0], Ops[2], Builder.CreateTrunc(Ops[1], Builder.getInt32Ty())}); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVCreateBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVectorImpl<Value *> &Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + llvm::Value *ReturnVector = llvm::PoisonValue::get(ResultType); + auto *VecTy = cast<ScalableVectorType>(Ops[0]->getType()); + for (unsigned I = 0, N = Ops.size(); I < N; ++I) { + if (isa<ScalableVectorType>(ResultType)) { + llvm::Value *Idx = ConstantInt::get(Builder.getInt64Ty(), + VecTy->getMinNumElements() * I); + ReturnVector = + Builder.CreateInsertVector(ResultType, ReturnVector, Ops[I], Idx); + } else { + llvm::Value *Idx = ConstantInt::get(Builder.getInt32Ty(), I); + ReturnVector = Builder.CreateIntrinsic(Intrinsic::riscv_tuple_insert, + {ResultType, Ops[I]->getType()}, + {ReturnVector, Ops[I], Idx}); + } + } + return ReturnVector; +} + Value *CodeGenFunction::EmitRISCVCpuInit() { llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {VoidPtrTy}, false); llvm::FunctionCallee Func = @@ -180,10 +1125,6 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, } Intrinsic::ID ID = Intrinsic::not_intrinsic; - // The 0th bit simulates the `vta` of RVV - // The 1st bit simulates the `vma` of RVV - constexpr unsigned RVV_VTA = 0x1; - constexpr unsigned RVV_VMA = 0x2; int PolicyAttrs = 0; bool IsMasked = false; // This is used by segment load/store to determine it's llvm type. diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp index 277d69daf493..af711c14d4ed 100644 --- a/clang/lib/CodeGen/TargetInfo.cpp +++ b/clang/lib/CodeGen/TargetInfo.cpp @@ -75,6 +75,8 @@ TargetCodeGenInfo::~TargetCodeGenInfo() = default; // If someone can figure out a general rule for this, that would be great. // It's probably just doomed to be platform-dependent, though. unsigned TargetCodeGenInfo::getSizeOfUnwindException() const { + if (getABIInfo().getContext().getLangOpts().hasSEHExceptions()) + return getABIInfo().getDataLayout().getPointerSizeInBits() > 32 ? 64 : 48; // Verified for: // x86-64 FreeBSD, Linux, Darwin // x86-32 FreeBSD, Linux, Darwin diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp index 9642196b78c6..0461f121d76c 100644 --- a/clang/lib/CodeGen/Targets/Sparc.cpp +++ b/clang/lib/CodeGen/Targets/Sparc.cpp @@ -8,6 +8,7 @@ #include "ABIInfoImpl.h" #include "TargetInfo.h" +#include <algorithm> using namespace clang; using namespace clang::CodeGen; @@ -109,7 +110,8 @@ public: SparcV9ABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {} private: - ABIArgInfo classifyType(QualType RetTy, unsigned SizeLimit) const; + ABIArgInfo classifyType(QualType RetTy, unsigned SizeLimit, + unsigned &RegOffset) const; void computeInfo(CGFunctionInfo &FI) const override; RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, AggValueSlot Slot) const override; @@ -222,127 +224,114 @@ private: }; } // end anonymous namespace -ABIArgInfo -SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { +ABIArgInfo SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit, + unsigned &RegOffset) const { if (Ty->isVoidType()) return ABIArgInfo::getIgnore(); - uint64_t Size = getContext().getTypeSize(Ty); + auto &Context = getContext(); + auto &VMContext = getVMContext(); + + uint64_t Size = Context.getTypeSize(Ty); + unsigned Alignment = Context.getTypeAlign(Ty); + bool NeedPadding = (Alignment > 64) && (RegOffset % 2 != 0); // Anything too big to fit in registers is passed with an explicit indirect // pointer / sret pointer. - if (Size > SizeLimit) + if (Size > SizeLimit) { + RegOffset += 1; return getNaturalAlignIndirect( Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false); + } // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); // Integer types smaller than a register are extended. - if (Size < 64 && Ty->isIntegerType()) + if (Size < 64 && Ty->isIntegerType()) { + RegOffset += 1; return ABIArgInfo::getExtend(Ty); + } if (const auto *EIT = Ty->getAs<BitIntType>()) - if (EIT->getNumBits() < 64) + if (EIT->getNumBits() < 64) { + RegOffset += 1; return ABIArgInfo::getExtend(Ty); + } // Other non-aggregates go in registers. - if (!isAggregateTypeForABI(Ty)) + if (!isAggregateTypeForABI(Ty)) { + RegOffset += Size / 64; return ABIArgInfo::getDirect(); + } // If a C++ object has either a non-trivial copy constructor or a non-trivial // destructor, it is passed with an explicit indirect pointer / sret pointer. - if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) + if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { + RegOffset += 1; return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(), RAA == CGCXXABI::RAA_DirectInMemory); + } // This is a small aggregate type that should be passed in registers. // Build a coercion type from the LLVM struct type. llvm::StructType *StrTy = dyn_cast<llvm::StructType>(CGT.ConvertType(Ty)); - if (!StrTy) + if (!StrTy) { + RegOffset += Size / 64; return ABIArgInfo::getDirect(); + } - CoerceBuilder CB(getVMContext(), getDataLayout()); + CoerceBuilder CB(VMContext, getDataLayout()); CB.addStruct(0, StrTy); // All structs, even empty ones, should take up a register argument slot, // so pin the minimum struct size to one bit. CB.pad(llvm::alignTo( std::max(CB.DL.getTypeSizeInBits(StrTy).getKnownMinValue(), uint64_t(1)), 64)); + RegOffset += CB.Size / 64; + + // If we're dealing with overaligned structs we may need to add a padding in + // the front, to preserve the correct register-memory mapping. + // + // See SCD 2.4.1, pages 3P-11 and 3P-12. + llvm::Type *Padding = + NeedPadding ? llvm::Type::getInt64Ty(VMContext) : nullptr; + RegOffset += NeedPadding ? 1 : 0; // Try to use the original type for coercion. llvm::Type *CoerceTy = CB.isUsableType(StrTy) ? StrTy : CB.getType(); - if (CB.InReg) - return ABIArgInfo::getDirectInReg(CoerceTy); - else - return ABIArgInfo::getDirect(CoerceTy); + ABIArgInfo AAI = ABIArgInfo::getDirect(CoerceTy, 0, Padding); + AAI.setInReg(CB.InReg); + return AAI; } RValue SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, AggValueSlot Slot) const { - ABIArgInfo AI = classifyType(Ty, 16 * 8); - llvm::Type *ArgTy = CGT.ConvertType(Ty); - if (AI.canHaveCoerceToType() && !AI.getCoerceToType()) - AI.setCoerceToType(ArgTy); - CharUnits SlotSize = CharUnits::fromQuantity(8); + auto TInfo = getContext().getTypeInfoInChars(Ty); - CGBuilderTy &Builder = CGF.Builder; - Address Addr = Address(Builder.CreateLoad(VAListAddr, "ap.cur"), - getVAListElementType(CGF), SlotSize); - llvm::Type *ArgPtrTy = CGF.UnqualPtrTy; - - auto TypeInfo = getContext().getTypeInfoInChars(Ty); - - Address ArgAddr = Address::invalid(); - CharUnits Stride; - switch (AI.getKind()) { - case ABIArgInfo::Expand: - case ABIArgInfo::CoerceAndExpand: - case ABIArgInfo::InAlloca: - llvm_unreachable("Unsupported ABI kind for va_arg"); - - case ABIArgInfo::Extend: { - Stride = SlotSize; - CharUnits Offset = SlotSize - TypeInfo.Width; - ArgAddr = Builder.CreateConstInBoundsByteGEP(Addr, Offset, "extend"); - break; - } - - case ABIArgInfo::Direct: { - auto AllocSize = getDataLayout().getTypeAllocSize(AI.getCoerceToType()); - Stride = CharUnits::fromQuantity(AllocSize).alignTo(SlotSize); - ArgAddr = Addr; - break; - } - - case ABIArgInfo::Indirect: - case ABIArgInfo::IndirectAliased: - Stride = SlotSize; - ArgAddr = Addr.withElementType(ArgPtrTy); - ArgAddr = Address(Builder.CreateLoad(ArgAddr, "indirect.arg"), ArgTy, - TypeInfo.Align); - break; + // Zero-sized types have a width of one byte for parameter passing purposes. + TInfo.Width = std::max(TInfo.Width, CharUnits::fromQuantity(1)); - case ABIArgInfo::Ignore: - return Slot.asRValue(); - } - - // Update VAList. - Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next"); - Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); - - return CGF.EmitLoadOfAnyValue( - CGF.MakeAddrLValue(ArgAddr.withElementType(ArgTy), Ty), Slot); + // Arguments bigger than 2*SlotSize bytes are passed indirectly. + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, + /*IsIndirect=*/TInfo.Width > 2 * SlotSize, TInfo, + SlotSize, + /*AllowHigherAlign=*/true, Slot); } void SparcV9ABIInfo::computeInfo(CGFunctionInfo &FI) const { - FI.getReturnInfo() = classifyType(FI.getReturnType(), 32 * 8); + unsigned RetOffset = 0; + ABIArgInfo RetType = classifyType(FI.getReturnType(), 32 * 8, RetOffset); + FI.getReturnInfo() = RetType; + + // Indirect returns will have its pointer passed as an argument. + unsigned ArgOffset = RetType.isIndirect() ? RetOffset : 0; for (auto &I : FI.arguments()) - I.info = classifyType(I.type, 16 * 8); + I.info = classifyType(I.type, 16 * 8, ArgOffset); } namespace { diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp index 3f9b808b2722..07a3ae925f96 100644 --- a/clang/lib/Driver/ToolChain.cpp +++ b/clang/lib/Driver/ToolChain.cpp @@ -837,17 +837,30 @@ void ToolChain::addFortranRuntimeLibs(const ArgList &Args, void ToolChain::addFortranRuntimeLibraryPath(const llvm::opt::ArgList &Args, ArgStringList &CmdArgs) const { - // Default to the <driver-path>/../lib directory. This works fine on the - // platforms that we have tested so far. We will probably have to re-fine - // this in the future. In particular, on some platforms, we may need to use - // lib64 instead of lib. + auto AddLibSearchPathIfExists = [&](const Twine &Path) { + // Linker may emit warnings about non-existing directories + if (!llvm::sys::fs::is_directory(Path)) + return; + + if (getTriple().isKnownWindowsMSVCEnvironment()) + CmdArgs.push_back(Args.MakeArgString("-libpath:" + Path)); + else + CmdArgs.push_back(Args.MakeArgString("-L" + Path)); + }; + + // Search for flang_rt.* at the same location as clang_rt.* with + // LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=0. On most platforms, flang_rt is + // located at the path returned by getRuntimePath() which is already added to + // the library search path. This exception is for Apple-Darwin. + AddLibSearchPathIfExists(getCompilerRTPath()); + + // Fall back to the non-resource directory <driver-path>/../lib. We will + // probably have to refine this in the future. In particular, on some + // platforms, we may need to use lib64 instead of lib. SmallString<256> DefaultLibPath = llvm::sys::path::parent_path(getDriver().Dir); llvm::sys::path::append(DefaultLibPath, "lib"); - if (getTriple().isKnownWindowsMSVCEnvironment()) - CmdArgs.push_back(Args.MakeArgString("-libpath:" + DefaultLibPath)); - else - CmdArgs.push_back(Args.MakeArgString("-L" + DefaultLibPath)); + AddLibSearchPathIfExists(DefaultLibPath); } void ToolChain::addFlangRTLibPath(const ArgList &Args, diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp index 6bd710ec6b8b..418f9fd9ca4c 100644 --- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp +++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp @@ -467,3 +467,18 @@ void aarch64::setPAuthABIInTriple(const Driver &D, const ArgList &Args, break; } } + +/// Is the triple {aarch64.aarch64_be}-none-elf? +bool aarch64::isAArch64BareMetal(const llvm::Triple &Triple) { + if (Triple.getArch() != llvm::Triple::aarch64 && + Triple.getArch() != llvm::Triple::aarch64_be) + return false; + + if (Triple.getVendor() != llvm::Triple::UnknownVendor) + return false; + + if (Triple.getOS() != llvm::Triple::UnknownOS) + return false; + + return Triple.getEnvironmentName() == "elf"; +} diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.h b/clang/lib/Driver/ToolChains/Arch/AArch64.h index 2057272867a1..2765ee8c3a6a 100644 --- a/clang/lib/Driver/ToolChains/Arch/AArch64.h +++ b/clang/lib/Driver/ToolChains/Arch/AArch64.h @@ -30,6 +30,7 @@ std::string getAArch64TargetCPU(const llvm::opt::ArgList &Args, void setPAuthABIInTriple(const Driver &D, const llvm::opt::ArgList &Args, llvm::Triple &triple); +bool isAArch64BareMetal(const llvm::Triple &Triple); } // end namespace aarch64 } // end namespace target diff --git a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp index 9595ee8383c8..94a94f1e9c48 100644 --- a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp +++ b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp @@ -23,7 +23,9 @@ const char *sparc::getSparcAsmModeForCPU(StringRef Name, if (Triple.getArch() == llvm::Triple::sparcv9) { const char *DefV9CPU; - if (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD()) + if (Triple.isOSSolaris()) + DefV9CPU = "-Av9b"; + else if (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD()) DefV9CPU = "-Av9a"; else DefV9CPU = "-Av9"; @@ -35,6 +37,13 @@ const char *sparc::getSparcAsmModeForCPU(StringRef Name, .Case("niagara4", "-Av9d") .Default(DefV9CPU); } else { + const char *DefV8CPU; + + if (Triple.isOSSolaris()) + DefV8CPU = "-Av8plus"; + else + DefV8CPU = "-Av8"; + return llvm::StringSwitch<const char *>(Name) .Case("v8", "-Av8") .Case("supersparc", "-Av8") @@ -70,7 +79,7 @@ const char *sparc::getSparcAsmModeForCPU(StringRef Name, .Case("gr712rc", "-Aleon") .Case("leon4", "-Aleon") .Case("gr740", "-Aleon") - .Default("-Av8"); + .Default(DefV8CPU); } } @@ -130,7 +139,8 @@ std::string sparc::getSparcTargetCPU(const Driver &D, const ArgList &Args, return ""; } -void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args, +void sparc::getSparcTargetFeatures(const Driver &D, const llvm::Triple &Triple, + const ArgList &Args, std::vector<StringRef> &Features) { sparc::FloatABI FloatABI = sparc::getSparcFloatABI(D, Args); if (FloatABI == sparc::FloatABI::Soft) @@ -150,11 +160,22 @@ void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args, Features.push_back("-popc"); } + // Those OSes default to enabling VIS on 64-bit SPARC. + // See also the corresponding code for external assemblers in + // sparc::getSparcAsmModeForCPU(). + bool IsSparcV9ATarget = + (Triple.getArch() == llvm::Triple::sparcv9) && + (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD()); + bool IsSparcV9BTarget = Triple.isOSSolaris(); + bool IsSparcV8PlusTarget = + Triple.getArch() == llvm::Triple::sparc && Triple.isOSSolaris(); if (Arg *A = Args.getLastArg(options::OPT_mvis, options::OPT_mno_vis)) { if (A->getOption().matches(options::OPT_mvis)) Features.push_back("+vis"); else Features.push_back("-vis"); + } else if (IsSparcV9ATarget) { + Features.push_back("+vis"); } if (Arg *A = Args.getLastArg(options::OPT_mvis2, options::OPT_mno_vis2)) { @@ -162,6 +183,8 @@ void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args, Features.push_back("+vis2"); else Features.push_back("-vis2"); + } else if (IsSparcV9BTarget) { + Features.push_back("+vis2"); } if (Arg *A = Args.getLastArg(options::OPT_mvis3, options::OPT_mno_vis3)) { @@ -182,6 +205,8 @@ void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args, if (Arg *A = Args.getLastArg(options::OPT_mv8plus, options::OPT_mno_v8plus)) { if (A->getOption().matches(options::OPT_mv8plus)) Features.push_back("+v8plus"); + } else if (IsSparcV8PlusTarget) { + Features.push_back("+v8plus"); } if (Args.hasArg(options::OPT_ffixed_g1)) diff --git a/clang/lib/Driver/ToolChains/Arch/Sparc.h b/clang/lib/Driver/ToolChains/Arch/Sparc.h index 2b178d9df1ee..fa25b4992cc8 100644 --- a/clang/lib/Driver/ToolChains/Arch/Sparc.h +++ b/clang/lib/Driver/ToolChains/Arch/Sparc.h @@ -31,7 +31,8 @@ FloatABI getSparcFloatABI(const Driver &D, const llvm::opt::ArgList &Args); std::string getSparcTargetCPU(const Driver &D, const llvm::opt::ArgList &Args, const llvm::Triple &Triple); -void getSparcTargetFeatures(const Driver &D, const llvm::opt::ArgList &Args, +void getSparcTargetFeatures(const Driver &D, const llvm::Triple &Triple, + const llvm::opt::ArgList &Args, std::vector<llvm::StringRef> &Features); const char *getSparcAsmModeForCPU(llvm::StringRef Name, const llvm::Triple &Triple); diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp index e670696cd59a..207150ea6f32 100644 --- a/clang/lib/Driver/ToolChains/BareMetal.cpp +++ b/clang/lib/Driver/ToolChains/BareMetal.cpp @@ -12,6 +12,7 @@ #include "clang/Driver/CommonArgs.h" #include "clang/Driver/InputInfo.h" +#include "Arch/AArch64.h" #include "Arch/ARM.h" #include "Arch/RISCV.h" #include "clang/Driver/Compilation.h" @@ -31,21 +32,6 @@ using namespace clang::driver; using namespace clang::driver::tools; using namespace clang::driver::toolchains; -/// Is the triple {aarch64.aarch64_be}-none-elf? -static bool isAArch64BareMetal(const llvm::Triple &Triple) { - if (Triple.getArch() != llvm::Triple::aarch64 && - Triple.getArch() != llvm::Triple::aarch64_be) - return false; - - if (Triple.getVendor() != llvm::Triple::UnknownVendor) - return false; - - if (Triple.getOS() != llvm::Triple::UnknownOS) - return false; - - return Triple.getEnvironmentName() == "elf"; -} - static bool isRISCVBareMetal(const llvm::Triple &Triple) { if (!Triple.isRISCV()) return false; @@ -363,8 +349,9 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple, } bool BareMetal::handlesTarget(const llvm::Triple &Triple) { - return arm::isARMEABIBareMetal(Triple) || isAArch64BareMetal(Triple) || - isRISCVBareMetal(Triple) || isPPCBareMetal(Triple); + return arm::isARMEABIBareMetal(Triple) || + aarch64::isAArch64BareMetal(Triple) || isRISCVBareMetal(Triple) || + isPPCBareMetal(Triple); } Tool *BareMetal::buildLinker() const { @@ -694,9 +681,6 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA, NeedCRTs) CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath(CRTEnd))); - if (TC.getTriple().isRISCV()) - CmdArgs.push_back("-X"); - // The R_ARM_TARGET2 relocation must be treated as R_ARM_REL32 on arm*-*-elf // and arm*-*-eabi (the default is R_ARM_GOT_PREL, used on arm*-*-linux and // arm*-*-*bsd). diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 456bfe885f35..62613322320c 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1752,7 +1752,6 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args, options::OPT_fno_ptrauth_objc_interface_sel); Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_objc_class_ro, options::OPT_fno_ptrauth_objc_class_ro); - if (Triple.getEnvironment() == llvm::Triple::PAuthTest) handlePAuthABI(Args, CmdArgs); @@ -2731,16 +2730,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C, CmdArgs.push_back(MipsTargetFeature); } - // Those OSes default to enabling VIS on 64-bit SPARC. - // See also the corresponding code for external assemblers in - // sparc::getSparcAsmModeForCPU(). - bool IsSparcV9ATarget = - (C.getDefaultToolChain().getArch() == llvm::Triple::sparcv9) && - (Triple.isOSLinux() || Triple.isOSFreeBSD() || Triple.isOSOpenBSD()); - if (IsSparcV9ATarget && SparcTargetFeatures.empty()) { - CmdArgs.push_back("-target-feature"); - CmdArgs.push_back("+vis"); - } for (const char *Feature : SparcTargetFeatures) { CmdArgs.push_back("-target-feature"); CmdArgs.push_back(Feature); diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 097d186ad8ea..8d3775de9be5 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -547,15 +547,22 @@ const char *tools::getLDMOption(const llvm::Triple &T, const ArgList &Args) { case llvm::Triple::aarch64: if (T.isOSManagarm()) return "aarch64managarm"; + else if (aarch64::isAArch64BareMetal(T)) + return "aarch64elf"; return "aarch64linux"; case llvm::Triple::aarch64_be: + if (aarch64::isAArch64BareMetal(T)) + return "aarch64elfb"; return "aarch64linuxb"; case llvm::Triple::arm: case llvm::Triple::thumb: case llvm::Triple::armeb: - case llvm::Triple::thumbeb: - return tools::arm::isARMBigEndian(T, Args) ? "armelfb_linux_eabi" - : "armelf_linux_eabi"; + case llvm::Triple::thumbeb: { + bool IsBigEndian = tools::arm::isARMBigEndian(T, Args); + if (arm::isARMEABIBareMetal(T)) + return IsBigEndian ? "armelfb" : "armelf"; + return IsBigEndian ? "armelfb_linux_eabi" : "armelf_linux_eabi"; + } case llvm::Triple::m68k: return "m68kelf"; case llvm::Triple::ppc: @@ -856,7 +863,7 @@ void tools::getTargetFeatures(const Driver &D, const llvm::Triple &Triple, case llvm::Triple::sparc: case llvm::Triple::sparcel: case llvm::Triple::sparcv9: - sparc::getSparcTargetFeatures(D, Args, Features); + sparc::getSparcTargetFeatures(D, Triple, Args, Features); break; case llvm::Triple::r600: case llvm::Triple::amdgcn: @@ -1320,6 +1327,17 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, if (Args.hasArg(options::OPT_ftime_report)) CmdArgs.push_back( Args.MakeArgString(Twine(PluginOptPrefix) + "-time-passes")); + + if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) { + CmdArgs.push_back( + Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue()))); + CmdArgs.push_back( + Args.MakeArgString("--thinlto-remote-compiler=" + + Twine(ToolChain.getDriver().getClangProgramPath()))); + + for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ)) + CmdArgs.push_back(Args.MakeArgString("--thinlto-distributor-arg=" + A)); + } } void tools::addOpenMPRuntimeLibraryPath(const ToolChain &TC, @@ -3247,14 +3265,8 @@ void tools::handleVectorizeSLPArgs(const ArgList &Args, void tools::handleInterchangeLoopsArgs(const ArgList &Args, ArgStringList &CmdArgs) { - // FIXME: instead of relying on shouldEnableVectorizerAtOLevel, we may want to - // implement a separate function to infer loop interchange from opt level. - // For now, enable loop-interchange at the same opt levels as loop-vectorize. - bool EnableInterchange = shouldEnableVectorizerAtOLevel(Args, false); - OptSpecifier InterchangeAliasOption = - EnableInterchange ? options::OPT_O_Group : options::OPT_floop_interchange; - if (Args.hasFlag(options::OPT_floop_interchange, InterchangeAliasOption, - options::OPT_fno_loop_interchange, EnableInterchange)) + if (Args.hasFlag(options::OPT_floop_interchange, + options::OPT_fno_loop_interchange, false)) CmdArgs.push_back("-floop-interchange"); } diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp index e5075cbcaf66..234683f2f488 100644 --- a/clang/lib/Driver/ToolChains/Darwin.cpp +++ b/clang/lib/Driver/ToolChains/Darwin.cpp @@ -3187,28 +3187,46 @@ void MachO::addClangTargetOptions(const llvm::opt::ArgList &DriverArgs, ToolChain::addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadKind); - // On arm64e, enable pointer authentication (for the return address and - // indirect calls), as well as usage of the intrinsics. - if (getArchName() == "arm64e") { + // On arm64e, we enable all the features required for the Darwin userspace + // ABI + if (getTriple().isArm64e()) { + // Core platform ABI + if (!DriverArgs.hasArg(options::OPT_fptrauth_calls, + options::OPT_fno_ptrauth_calls)) + CC1Args.push_back("-fptrauth-calls"); if (!DriverArgs.hasArg(options::OPT_fptrauth_returns, options::OPT_fno_ptrauth_returns)) CC1Args.push_back("-fptrauth-returns"); - if (!DriverArgs.hasArg(options::OPT_fptrauth_intrinsics, options::OPT_fno_ptrauth_intrinsics)) CC1Args.push_back("-fptrauth-intrinsics"); - - if (!DriverArgs.hasArg(options::OPT_fptrauth_calls, - options::OPT_fno_ptrauth_calls)) - CC1Args.push_back("-fptrauth-calls"); - if (!DriverArgs.hasArg(options::OPT_fptrauth_indirect_gotos, options::OPT_fno_ptrauth_indirect_gotos)) CC1Args.push_back("-fptrauth-indirect-gotos"); - if (!DriverArgs.hasArg(options::OPT_fptrauth_auth_traps, options::OPT_fno_ptrauth_auth_traps)) CC1Args.push_back("-fptrauth-auth-traps"); + + // C++ v-table ABI + if (!DriverArgs.hasArg( + options::OPT_fptrauth_vtable_pointer_address_discrimination, + options::OPT_fno_ptrauth_vtable_pointer_address_discrimination)) + CC1Args.push_back("-fptrauth-vtable-pointer-address-discrimination"); + if (!DriverArgs.hasArg( + options::OPT_fptrauth_vtable_pointer_type_discrimination, + options::OPT_fno_ptrauth_vtable_pointer_type_discrimination)) + CC1Args.push_back("-fptrauth-vtable-pointer-type-discrimination"); + + // Objective-C ABI + if (!DriverArgs.hasArg(options::OPT_fptrauth_objc_isa, + options::OPT_fno_ptrauth_objc_isa)) + CC1Args.push_back("-fptrauth-objc-isa"); + if (!DriverArgs.hasArg(options::OPT_fptrauth_objc_class_ro, + options::OPT_fno_ptrauth_objc_class_ro)) + CC1Args.push_back("-fptrauth-objc-class-ro"); + if (!DriverArgs.hasArg(options::OPT_fptrauth_objc_interface_sel, + options::OPT_fno_ptrauth_objc_interface_sel)) + CC1Args.push_back("-fptrauth-objc-interface-sel"); } } diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp index 1edb83f7255e..7ab41e9b85a0 100644 --- a/clang/lib/Driver/ToolChains/Flang.cpp +++ b/clang/lib/Driver/ToolChains/Flang.cpp @@ -447,6 +447,7 @@ void Flang::addTargetOptions(const ArgList &Args, // Add the target features. switch (TC.getArch()) { default: + getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false); break; case llvm::Triple::aarch64: getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false); diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp index b2e36ae6f97c..4894e6a52e93 100644 --- a/clang/lib/Driver/ToolChains/MinGW.cpp +++ b/clang/lib/Driver/ToolChains/MinGW.cpp @@ -85,11 +85,18 @@ void tools::MinGW::Linker::AddLibGCC(const ArgList &Args, CmdArgs.push_back("-lmoldname"); CmdArgs.push_back("-lmingwex"); - for (auto Lib : Args.getAllArgValues(options::OPT_l)) + for (auto Lib : Args.getAllArgValues(options::OPT_l)) { if (StringRef(Lib).starts_with("msvcr") || StringRef(Lib).starts_with("ucrt") || - StringRef(Lib).starts_with("crtdll")) + StringRef(Lib).starts_with("crtdll")) { + std::string CRTLib = (llvm::Twine("-l") + Lib).str(); + // Respect the user's chosen crt variant, but still provide it + // again as the last linker argument, because some of the libraries + // we added above may depend on it. + CmdArgs.push_back(Args.MakeArgStringRef(CRTLib)); return; + } + } CmdArgs.push_back("-lmsvcrt"); } diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp index 79b1b6960da1..8f589186af34 100644 --- a/clang/lib/Driver/ToolChains/OpenBSD.cpp +++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp @@ -161,7 +161,7 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, if (Nopie || Profiling) CmdArgs.push_back("-nopie"); - if (Triple.isRISCV64()) { + if (Triple.isLoongArch64() || Triple.isRISCV64()) { CmdArgs.push_back("-X"); if (Args.hasArg(options::OPT_mno_relax)) CmdArgs.push_back("--no-relax"); diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp index 4010f7fbd25b..38104f6c78a6 100644 --- a/clang/lib/Format/ContinuationIndenter.cpp +++ b/clang/lib/Format/ContinuationIndenter.cpp @@ -411,7 +411,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { } if (CurrentState.BreakBeforeClosingBrace && (Current.closesBlockOrBlockTypeList(Style) || - (Current.is(tok::r_brace) && + (Current.is(tok::r_brace) && Current.MatchingParen && Current.isBlockIndentedInitRBrace(Style)))) { return true; } @@ -628,9 +628,16 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { // name. !Style.isJavaScript() && Previous.isNot(tok::kw_template) && CurrentState.BreakBeforeParameter) { - for (const auto *Tok = &Previous; Tok; Tok = Tok->Previous) - if (Tok->FirstAfterPPLine || Tok->is(TT_LineComment)) + for (const auto *Tok = &Previous; Tok; Tok = Tok->Previous) { + if (Tok->is(TT_LineComment)) + return false; + if (Tok->is(TT_TemplateCloser)) { + Tok = Tok->MatchingParen; + assert(Tok); + } + if (Tok->FirstAfterPPLine) return false; + } return true; } diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index 78c09be458f0..5bdb810a3925 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -727,6 +727,7 @@ template <> struct MappingTraits<FormatStyle::SpaceBeforeParensCustom> { IO.mapOptional("AfterFunctionDeclarationName", Spacing.AfterFunctionDeclarationName); IO.mapOptional("AfterIfMacros", Spacing.AfterIfMacros); + IO.mapOptional("AfterNot", Spacing.AfterNot); IO.mapOptional("AfterOverloadedOperator", Spacing.AfterOverloadedOperator); IO.mapOptional("AfterPlacementOperator", Spacing.AfterPlacementOperator); IO.mapOptional("AfterRequiresInClause", Spacing.AfterRequiresInClause); @@ -1748,7 +1749,6 @@ FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) { GoogleStyle.AttributeMacros.push_back("absl_nullable"); GoogleStyle.AttributeMacros.push_back("absl_nullability_unknown"); GoogleStyle.BreakTemplateDeclarations = FormatStyle::BTDS_Yes; - GoogleStyle.DerivePointerAlignment = true; GoogleStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup; GoogleStyle.IncludeStyle.IncludeCategories = {{"^<ext/.*\\.h>", 2, 0, false}, {"^<.*\\.h>", 1, 0, false}, @@ -1857,6 +1857,7 @@ FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) { } else if (Language == FormatStyle::LK_ObjC) { GoogleStyle.AlwaysBreakBeforeMultilineStrings = false; GoogleStyle.ColumnLimit = 100; + GoogleStyle.DerivePointerAlignment = true; // "Regroup" doesn't work well for ObjC yet (main header heuristic, // relationship between ObjC standard library headers and other heades, // #imports, etc.) @@ -2131,48 +2132,70 @@ std::error_code parseConfiguration(llvm::MemoryBufferRef Config, Input >> Styles; if (Input.error()) return Input.error(); + if (Styles.empty()) + return make_error_code(ParseError::Success); + + const auto StyleCount = Styles.size(); - for (unsigned i = 0; i < Styles.size(); ++i) { - // Ensures that only the first configuration can skip the Language option. - if (Styles[i].Language == FormatStyle::LK_None && i != 0) + // Start from the second style as (only) the first one may be the default. + for (unsigned I = 1; I < StyleCount; ++I) { + const auto Lang = Styles[I].Language; + if (Lang == FormatStyle::LK_None) return make_error_code(ParseError::Error); // Ensure that each language is configured at most once. - for (unsigned j = 0; j < i; ++j) { - if (Styles[i].Language == Styles[j].Language) { + for (unsigned J = 0; J < I; ++J) { + if (Lang == Styles[J].Language) { LLVM_DEBUG(llvm::dbgs() << "Duplicate languages in the config file on positions " - << j << " and " << i << "\n"); + << J << " and " << I << '\n'); return make_error_code(ParseError::Error); } } } - // Look for a suitable configuration starting from the end, so we can - // find the configuration for the specific language first, and the default - // configuration (which can only be at slot 0) after it. - FormatStyle::FormatStyleSet StyleSet; - bool LanguageFound = false; - for (const FormatStyle &Style : llvm::reverse(Styles)) { - const auto Lang = Style.Language; - if (Lang != FormatStyle::LK_None) - StyleSet.Add(Style); - if (Lang == Language || - // For backward compatibility. - (Lang == FormatStyle::LK_Cpp && Language == FormatStyle::LK_C)) { - LanguageFound = true; - } else if (IsDotHFile && Language == FormatStyle::LK_Cpp && - (Lang == FormatStyle::LK_C || Lang == FormatStyle::LK_ObjC)) { - Language = Lang; - LanguageFound = true; + + int LanguagePos = -1; // Position of the style for Language. + int CppPos = -1; // Position of the style for C++. + int CPos = -1; // Position of the style for C. + + // Search Styles for Language and store the positions of C++ and C styles in + // case Language is not found. + for (unsigned I = 0; I < StyleCount; ++I) { + const auto Lang = Styles[I].Language; + if (Lang == Language) { + LanguagePos = I; + break; } - } - if (!LanguageFound) { - if (Styles.empty() || Styles[0].Language != FormatStyle::LK_None) + if (Lang == FormatStyle::LK_Cpp) + CppPos = I; + else if (Lang == FormatStyle::LK_C) + CPos = I; + } + + // If Language is not found, use the default style if there is one. Otherwise, + // use the C style for C++ .h files and for backward compatibility, the C++ + // style for .c files. + if (LanguagePos < 0) { + if (Styles[0].Language == FormatStyle::LK_None) // Default style. + LanguagePos = 0; + else if (IsDotHFile && Language == FormatStyle::LK_Cpp) + LanguagePos = CPos; + else if (!IsDotHFile && Language == FormatStyle::LK_C) + LanguagePos = CppPos; + if (LanguagePos < 0) return make_error_code(ParseError::Unsuitable); - FormatStyle DefaultStyle = Styles[0]; - DefaultStyle.Language = Language; - StyleSet.Add(std::move(DefaultStyle)); } - *Style = *StyleSet.Get(Language); + + for (const auto &S : llvm::reverse(llvm::drop_begin(Styles))) + Style->StyleSet.Add(S); + + *Style = Styles[LanguagePos]; + + if (LanguagePos == 0) { + if (Style->Language == FormatStyle::LK_None) // Default style. + Style->Language = Language; + Style->StyleSet.Add(*Style); + } + if (Style->InsertTrailingCommas != FormatStyle::TCS_None && Style->BinPackArguments) { // See comment on FormatStyle::TSC_Wrapped. @@ -2203,14 +2226,8 @@ FormatStyle::FormatStyleSet::Get(FormatStyle::LanguageKind Language) const { if (!Styles) return std::nullopt; auto It = Styles->find(Language); - if (It == Styles->end()) { - if (Language != FormatStyle::LK_C) - return std::nullopt; - // For backward compatibility. - It = Styles->find(FormatStyle::LK_Cpp); - if (It == Styles->end()) - return std::nullopt; - } + if (It == Styles->end()) + return std::nullopt; FormatStyle Style = It->second; Style.StyleSet = *this; return Style; @@ -4042,6 +4059,7 @@ LangOptions getFormattingLangOpts(const FormatStyle &Style) { switch (Style.Language) { case FormatStyle::LK_C: LangOpts.C11 = 1; + LangOpts.C23 = 1; break; case FormatStyle::LK_Cpp: case FormatStyle::LK_ObjC: diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp index 0d8ae1c4a77e..a32175ecc02b 100644 --- a/clang/lib/Format/FormatToken.cpp +++ b/clang/lib/Format/FormatToken.cpp @@ -53,12 +53,13 @@ bool FormatToken::isTypeOrIdentifier(const LangOptions &LangOpts) const { bool FormatToken::isBlockIndentedInitRBrace(const FormatStyle &Style) const { assert(is(tok::r_brace)); + assert(MatchingParen); + assert(MatchingParen->is(tok::l_brace)); if (!Style.Cpp11BracedListStyle || Style.AlignAfterOpenBracket != FormatStyle::BAS_BlockIndent) { return false; } const auto *LBrace = MatchingParen; - assert(LBrace && LBrace->is(tok::l_brace)); if (LBrace->is(BK_BracedInit)) return true; if (LBrace->Previous && LBrace->Previous->is(tok::equal)) diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index 9252a795a0b5..c2000a971c89 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -55,7 +55,7 @@ namespace format { TYPE(ConflictAlternative) \ TYPE(ConflictEnd) \ TYPE(ConflictStart) \ - /* l_brace of if/for/while */ \ + /* l_brace of if/for/while/switch/catch */ \ TYPE(ControlStatementLBrace) \ TYPE(ControlStatementRBrace) \ TYPE(CppCastLParen) \ diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp index 49da3160daf5..165ede2d04b2 100644 --- a/clang/lib/Format/FormatTokenLexer.cpp +++ b/clang/lib/Format/FormatTokenLexer.cpp @@ -93,12 +93,6 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() { auto &Tok = *Tokens.back(); const auto NewlinesBefore = Tok.NewlinesBefore; switch (FormatOff) { - case FO_CurrentLine: - if (NewlinesBefore == 0) - Tok.Finalized = true; - else - FormatOff = FO_None; - break; case FO_NextLine: if (NewlinesBefore > 1) { FormatOff = FO_None; @@ -107,6 +101,13 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() { FormatOff = FO_CurrentLine; } break; + case FO_CurrentLine: + if (NewlinesBefore == 0) { + Tok.Finalized = true; + break; + } + FormatOff = FO_None; + [[fallthrough]]; default: if (!FormattingDisabled && FormatOffRegex.match(Tok.TokenText)) { if (Tok.is(tok::comment) && diff --git a/clang/lib/Format/IntegerLiteralSeparatorFixer.cpp b/clang/lib/Format/IntegerLiteralSeparatorFixer.cpp index 87823ae32b11..aa752f5e3148 100644 --- a/clang/lib/Format/IntegerLiteralSeparatorFixer.cpp +++ b/clang/lib/Format/IntegerLiteralSeparatorFixer.cpp @@ -45,15 +45,18 @@ std::pair<tooling::Replacements, unsigned> IntegerLiteralSeparatorFixer::process(const Environment &Env, const FormatStyle &Style) { switch (Style.Language) { - case FormatStyle::LK_Cpp: - case FormatStyle::LK_ObjC: - Separator = '\''; - break; case FormatStyle::LK_CSharp: case FormatStyle::LK_Java: case FormatStyle::LK_JavaScript: Separator = '_'; break; + case FormatStyle::LK_Cpp: + case FormatStyle::LK_ObjC: + if (Style.Standard >= FormatStyle::LS_Cpp14) { + Separator = '\''; + break; + } + [[fallthrough]]; default: return {}; } diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 739209a5681f..57b2872566a4 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -829,11 +829,6 @@ private: if (Parent && Parent->is(TT_PointerOrReference)) Parent->overwriteFixedType(TT_BinaryOperator); } - // An arrow after an ObjC method expression is not a lambda arrow. - if (CurrentToken->is(TT_ObjCMethodExpr) && CurrentToken->Next && - CurrentToken->Next->is(TT_LambdaArrow)) { - CurrentToken->Next->overwriteFixedType(TT_Unknown); - } Left->MatchingParen = CurrentToken; CurrentToken->MatchingParen = Left; // FirstObjCSelectorName is set when a colon is found. This does @@ -2590,6 +2585,9 @@ private: if (!Tok.Previous || Tok.isNot(tok::identifier) || Tok.is(TT_ClassHeadName)) return false; + if (Tok.endsSequence(Keywords.kw_final, TT_ClassHeadName)) + return false; + if ((Style.isJavaScript() || Style.isJava()) && Tok.is(Keywords.kw_extends)) return false; @@ -2996,14 +2994,18 @@ private: const FormatToken *PrevToken = Tok.getPreviousNonComment(); if (!PrevToken) return TT_UnaryOperator; - if (PrevToken->is(TT_TypeName)) + if (PrevToken->isTypeName(LangOpts)) return TT_PointerOrReference; if (PrevToken->isPlacementOperator() && Tok.is(tok::ampamp)) return TT_BinaryOperator; - const FormatToken *NextToken = Tok.getNextNonComment(); + auto *NextToken = Tok.getNextNonComment(); if (!NextToken) return TT_PointerOrReference; + if (NextToken->is(tok::greater)) { + NextToken->setFinalizedType(TT_TemplateCloser); + return TT_PointerOrReference; + } if (InTemplateArgument && NextToken->is(tok::kw_noexcept)) return TT_BinaryOperator; @@ -3112,7 +3114,7 @@ private: // It's more likely that & represents operator& than an uninitialized // reference. - if (Tok.is(tok::amp) && PrevToken && PrevToken->Tok.isAnyIdentifier() && + if (Tok.is(tok::amp) && PrevToken->Tok.isAnyIdentifier() && IsChainedOperatorAmpOrMember(PrevToken->getPreviousNonComment()) && NextToken && NextToken->Tok.isAnyIdentifier()) { if (auto NextNext = NextToken->getNextNonComment(); @@ -3773,18 +3775,12 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts, if (Current.is(TT_FunctionDeclarationName)) return true; - if (!Current.Tok.getIdentifierInfo()) + if (!Current.isOneOf(tok::identifier, tok::kw_operator)) return false; const auto *Prev = Current.getPreviousNonComment(); assert(Prev); - if (Prev->is(tok::coloncolon)) - Prev = Prev->Previous; - - if (!Prev) - return false; - const auto &Previous = *Prev; if (const auto *PrevPrev = Previous.getPreviousNonComment(); @@ -3833,6 +3829,8 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts, // Find parentheses of parameter list. if (Current.is(tok::kw_operator)) { + if (Line.startsWith(tok::kw_friend)) + return true; if (Previous.Tok.getIdentifierInfo() && !Previous.isOneOf(tok::kw_return, tok::kw_co_return)) { return true; @@ -3998,29 +3996,28 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const { } } - if (IsCpp && - (LineIsFunctionDeclaration || - (FirstNonComment && FirstNonComment->is(TT_CtorDtorDeclName))) && - Line.endsWith(tok::semi, tok::r_brace)) { - auto *Tok = Line.Last->Previous; - while (Tok->isNot(tok::r_brace)) - Tok = Tok->Previous; - if (auto *LBrace = Tok->MatchingParen; LBrace) { - assert(LBrace->is(tok::l_brace)); - Tok->setBlockKind(BK_Block); - LBrace->setBlockKind(BK_Block); - LBrace->setFinalizedType(TT_FunctionLBrace); + if (IsCpp) { + if ((LineIsFunctionDeclaration || + (FirstNonComment && FirstNonComment->is(TT_CtorDtorDeclName))) && + Line.endsWith(tok::semi, tok::r_brace)) { + auto *Tok = Line.Last->Previous; + while (Tok->isNot(tok::r_brace)) + Tok = Tok->Previous; + if (auto *LBrace = Tok->MatchingParen; LBrace && LBrace->is(TT_Unknown)) { + assert(LBrace->is(tok::l_brace)); + Tok->setBlockKind(BK_Block); + LBrace->setBlockKind(BK_Block); + LBrace->setFinalizedType(TT_FunctionLBrace); + } } - } - if (IsCpp && SeenName && AfterLastAttribute && - mustBreakAfterAttributes(*AfterLastAttribute, Style)) { - AfterLastAttribute->MustBreakBefore = true; - if (LineIsFunctionDeclaration) - Line.ReturnTypeWrapped = true; - } + if (SeenName && AfterLastAttribute && + mustBreakAfterAttributes(*AfterLastAttribute, Style)) { + AfterLastAttribute->MustBreakBefore = true; + if (LineIsFunctionDeclaration) + Line.ReturnTypeWrapped = true; + } - if (IsCpp) { if (!LineIsFunctionDeclaration) { // Annotate */&/&& in `operator` function calls as binary operators. for (const auto *Tok = FirstNonComment; Tok; Tok = Tok->Next) { @@ -4066,6 +4063,11 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const { } } + if (First->is(TT_ElseLBrace)) { + First->CanBreakBefore = true; + First->MustBreakBefore = true; + } + bool InFunctionDecl = Line.MightBeFunctionDecl; bool InParameterList = false; for (auto *Current = First->Next; Current; Current = Current->Next) { @@ -5474,7 +5476,8 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, if (Left.TokenText == "!") return Style.SpaceAfterLogicalNot; assert(Left.TokenText == "not"); - return Right.isOneOf(tok::coloncolon, TT_UnaryOperator); + return Right.isOneOf(tok::coloncolon, TT_UnaryOperator) || + (Right.is(tok::l_paren) && Style.SpaceBeforeParensOptions.AfterNot); } // If the next token is a binary operator or a selector name, we have @@ -6266,7 +6269,8 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, } if (Right.is(tok::colon) && - !Right.isOneOf(TT_CtorInitializerColon, TT_InlineASMColon)) { + !Right.isOneOf(TT_CtorInitializerColon, TT_InlineASMColon, + TT_BitFieldColon)) { return false; } if (Left.is(tok::colon) && Left.isOneOf(TT_DictLiteral, TT_ObjCMethodExpr)) { diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp index 91b8fdc8a3c3..934605733542 100644 --- a/clang/lib/Format/UnwrappedLineParser.cpp +++ b/clang/lib/Format/UnwrappedLineParser.cpp @@ -2266,7 +2266,7 @@ bool UnwrappedLineParser::tryToParseLambda() { if (!tryToParseLambdaIntroducer()) return false; - bool SeenArrow = false; + FormatToken *Arrow = nullptr; bool InTemplateParameterList = false; while (FormatTok->isNot(tok::l_brace)) { @@ -2341,17 +2341,13 @@ bool UnwrappedLineParser::tryToParseLambda() { case tok::ellipsis: case tok::kw_true: case tok::kw_false: - if (SeenArrow || InTemplateParameterList) { + if (Arrow || InTemplateParameterList) { nextToken(); break; } return true; case tok::arrow: - // This might or might not actually be a lambda arrow (this could be an - // ObjC method invocation followed by a dereferencing arrow). We might - // reset this back to TT_Unknown in TokenAnnotator. - FormatTok->setFinalizedType(TT_LambdaArrow); - SeenArrow = true; + Arrow = FormatTok; nextToken(); break; case tok::kw_requires: { @@ -2373,6 +2369,9 @@ bool UnwrappedLineParser::tryToParseLambda() { FormatTok->setFinalizedType(TT_LambdaLBrace); LSquare.setFinalizedType(TT_LambdaLSquare); + if (Arrow) + Arrow->setFinalizedType(TT_LambdaArrow); + NestedLambdas.push_back(Line->SeenDecltypeAuto); parseChildBlock(); assert(!NestedLambdas.empty()); @@ -2386,11 +2385,6 @@ bool UnwrappedLineParser::tryToParseLambdaIntroducer() { const FormatToken *LeftSquare = FormatTok; nextToken(); if (Previous) { - if (Previous->Tok.getIdentifierInfo() && - !Previous->isOneOf(tok::kw_return, tok::kw_co_await, tok::kw_co_yield, - tok::kw_co_return)) { - return false; - } if (Previous->closesScope()) { // Not a potential C-style cast. if (Previous->isNot(tok::r_paren)) @@ -2400,6 +2394,13 @@ bool UnwrappedLineParser::tryToParseLambdaIntroducer() { // and `int (*)()`. if (!BeforeRParen || !BeforeRParen->isOneOf(tok::greater, tok::r_paren)) return false; + } else if (Previous->is(tok::star)) { + Previous = Previous->getPreviousNonComment(); + } + if (Previous && Previous->Tok.getIdentifierInfo() && + !Previous->isOneOf(tok::kw_return, tok::kw_co_await, tok::kw_co_yield, + tok::kw_co_return)) { + return false; } } if (LeftSquare->isCppStructuredBinding(IsCpp)) diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 08cf0ae6b2c2..08f3b7a7fcc4 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -1542,6 +1542,17 @@ void CompilerInvocation::setDefaultPointerAuthOptions( Discrimination::Constant, InitFiniPointerConstantDiscriminator); } + Opts.BlockInvocationFunctionPointers = + PointerAuthSchema(Key::ASIA, true, Discrimination::None); + Opts.BlockHelperFunctionPointers = + PointerAuthSchema(Key::ASIA, true, Discrimination::None); + Opts.BlockByrefHelperFunctionPointers = + PointerAuthSchema(Key::ASIA, true, Discrimination::None); + if (LangOpts.PointerAuthBlockDescriptorPointers) + Opts.BlockDescriptorPointers = + PointerAuthSchema(Key::ASDA, true, Discrimination::Constant, + BlockDescriptorConstantDiscriminator); + Opts.ObjCMethodListFunctionPointers = PointerAuthSchema(Key::ASIA, true, Discrimination::None); Opts.ObjCMethodListPointer = @@ -3598,6 +3609,8 @@ static void GeneratePointerAuthArgs(const LangOptions &Opts, GenerateArg(Consumer, OPT_fptrauth_objc_interface_sel); if (Opts.PointerAuthObjcClassROPointers) GenerateArg(Consumer, OPT_fptrauth_objc_class_ro); + if (Opts.PointerAuthBlockDescriptorPointers) + GenerateArg(Consumer, OPT_fptrauth_block_descriptor_pointers); } static void ParsePointerAuthArgs(LangOptions &Opts, ArgList &Args, @@ -3621,7 +3634,8 @@ static void ParsePointerAuthArgs(LangOptions &Opts, ArgList &Args, Opts.PointerAuthELFGOT = Args.hasArg(OPT_fptrauth_elf_got); Opts.AArch64JumpTableHardening = Args.hasArg(OPT_faarch64_jump_table_hardening); - + Opts.PointerAuthBlockDescriptorPointers = + Args.hasArg(OPT_fptrauth_block_descriptor_pointers); Opts.PointerAuthObjcIsa = Args.hasArg(OPT_fptrauth_objc_isa); Opts.PointerAuthObjcClassROPointers = Args.hasArg(OPT_fptrauth_objc_class_ro); Opts.PointerAuthObjcInterfaceSel = diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp index 34fb825e9d42..cce8392950b0 100644 --- a/clang/lib/Frontend/InitPreprocessor.cpp +++ b/clang/lib/Frontend/InitPreprocessor.cpp @@ -1535,6 +1535,9 @@ static void InitializePredefinedMacros(const TargetInfo &TI, #undef TARGET_OS } + if (LangOpts.PointerAuthIntrinsics) + Builder.defineMacro("__PTRAUTH__"); + // Get other target #defines. TI.getTargetDefines(LangOpts, Builder); } diff --git a/clang/lib/Headers/avx10_2_512niintrin.h b/clang/lib/Headers/avx10_2_512niintrin.h index 7e614f7740bf..9d96e36c74ca 100644 --- a/clang/lib/Headers/avx10_2_512niintrin.h +++ b/clang/lib/Headers/avx10_2_512niintrin.h @@ -197,7 +197,7 @@ _mm512_mask_dpwsud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsud_epi32( - __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { + __mmask16 __U, __m512i __A, __m512i __B, __m512i __C) { return (__m512i)__builtin_ia32_selectd_512( (__mmask16)__U, (__v16si)_mm512_dpwsud_epi32(__A, __B, __C), (__v16si)_mm512_setzero_si512()); @@ -218,7 +218,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwsuds_epi32( } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsuds_epi32( - __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { + __mmask16 __U, __m512i __A, __m512i __B, __m512i __C) { return (__m512i)__builtin_ia32_selectd_512( (__mmask16)__U, (__v16si)_mm512_dpwsuds_epi32(__A, __B, __C), (__v16si)_mm512_setzero_si512()); @@ -239,7 +239,7 @@ _mm512_mask_dpwusd_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusd_epi32( - __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { + __mmask16 __U, __m512i __A, __m512i __B, __m512i __C) { return (__m512i)__builtin_ia32_selectd_512( (__mmask16)__U, (__v16si)_mm512_dpwusd_epi32(__A, __B, __C), (__v16si)_mm512_setzero_si512()); @@ -260,7 +260,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwusds_epi32( } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusds_epi32( - __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { + __mmask16 __U, __m512i __A, __m512i __B, __m512i __C) { return (__m512i)__builtin_ia32_selectd_512( (__mmask16)__U, (__v16si)_mm512_dpwusds_epi32(__A, __B, __C), (__v16si)_mm512_setzero_si512()); @@ -281,7 +281,7 @@ _mm512_mask_dpwuud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuud_epi32( - __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { + __mmask16 __U, __m512i __A, __m512i __B, __m512i __C) { return (__m512i)__builtin_ia32_selectd_512( (__mmask16)__U, (__v16si)_mm512_dpwuud_epi32(__A, __B, __C), (__v16si)_mm512_setzero_si512()); @@ -302,7 +302,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwuuds_epi32( } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuuds_epi32( - __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) { + __mmask16 __U, __m512i __A, __m512i __B, __m512i __C) { return (__m512i)__builtin_ia32_selectd_512( (__mmask16)__U, (__v16si)_mm512_dpwuuds_epi32(__A, __B, __C), (__v16si)_mm512_setzero_si512()); diff --git a/clang/lib/Headers/avx10_2bf16intrin.h b/clang/lib/Headers/avx10_2bf16intrin.h index 66797ae00fe4..0ca538082939 100644 --- a/clang/lib/Headers/avx10_2bf16intrin.h +++ b/clang/lib/Headers/avx10_2bf16intrin.h @@ -519,34 +519,34 @@ _mm_maskz_min_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) { (__mmask8)__U, (__v8bf)_mm_min_pbh(__A, __B), (__v8bf)_mm_setzero_pbh()); } -static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comieq_sbh(__m128bh A, - __m128bh B) { - return __builtin_ia32_vcomisbf16eq((__v8bf)A, (__v8bf)B); +static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comieq_sbh(__m128bh __A, + __m128bh __B) { + return __builtin_ia32_vcomisbf16eq((__v8bf)__A, (__v8bf)__B); } -static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comilt_sbh(__m128bh A, - __m128bh B) { - return __builtin_ia32_vcomisbf16lt((__v8bf)A, (__v8bf)B); +static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comilt_sbh(__m128bh __A, + __m128bh __B) { + return __builtin_ia32_vcomisbf16lt((__v8bf)__A, (__v8bf)__B); } -static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comile_sbh(__m128bh A, - __m128bh B) { - return __builtin_ia32_vcomisbf16le((__v8bf)A, (__v8bf)B); +static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comile_sbh(__m128bh __A, + __m128bh __B) { + return __builtin_ia32_vcomisbf16le((__v8bf)__A, (__v8bf)__B); } -static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comigt_sbh(__m128bh A, - __m128bh B) { - return __builtin_ia32_vcomisbf16gt((__v8bf)A, (__v8bf)B); +static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comigt_sbh(__m128bh __A, + __m128bh __B) { + return __builtin_ia32_vcomisbf16gt((__v8bf)__A, (__v8bf)__B); } -static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comige_sbh(__m128bh A, - __m128bh B) { - return __builtin_ia32_vcomisbf16ge((__v8bf)A, (__v8bf)B); +static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comige_sbh(__m128bh __A, + __m128bh __B) { + return __builtin_ia32_vcomisbf16ge((__v8bf)__A, (__v8bf)__B); } -static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comineq_sbh(__m128bh A, - __m128bh B) { - return __builtin_ia32_vcomisbf16neq((__v8bf)A, (__v8bf)B); +static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comineq_sbh(__m128bh __A, + __m128bh __B) { + return __builtin_ia32_vcomisbf16neq((__v8bf)__A, (__v8bf)__B); } #define _mm256_cmp_pbh_mask(__A, __B, __P) \ diff --git a/clang/lib/Headers/avx10_2niintrin.h b/clang/lib/Headers/avx10_2niintrin.h index 992be18f7720..d5a66cfef536 100644 --- a/clang/lib/Headers/avx10_2niintrin.h +++ b/clang/lib/Headers/avx10_2niintrin.h @@ -253,7 +253,7 @@ _mm_mask_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { +_mm_maskz_dpwsud_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) { return (__m128i)__builtin_ia32_selectd_128( (__mmask8)__U, (__v4si)_mm_dpwsud_epi32(__A, __B, __C), (__v4si)_mm_setzero_si128()); @@ -266,7 +266,7 @@ _mm256_mask_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { +_mm256_maskz_dpwsud_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) { return (__m256i)__builtin_ia32_selectd_256( (__mmask8)__U, (__v8si)_mm256_dpwsud_epi32(__A, __B, __C), (__v8si)_mm256_setzero_si256()); @@ -279,7 +279,7 @@ _mm_mask_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { +_mm_maskz_dpwsuds_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) { return (__m128i)__builtin_ia32_selectd_128( (__mmask8)__U, (__v4si)_mm_dpwsuds_epi32(__A, __B, __C), (__v4si)_mm_setzero_si128()); @@ -292,7 +292,7 @@ _mm256_mask_dpwsuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { } static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwsuds_epi32( - __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { + __mmask8 __U, __m256i __A, __m256i __B, __m256i __C) { return (__m256i)__builtin_ia32_selectd_256( (__mmask8)__U, (__v8si)_mm256_dpwsuds_epi32(__A, __B, __C), (__v8si)_mm256_setzero_si256()); @@ -305,7 +305,7 @@ _mm_mask_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { +_mm_maskz_dpwusd_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) { return (__m128i)__builtin_ia32_selectd_128( (__mmask8)__U, (__v4si)_mm_dpwusd_epi32(__A, __B, __C), (__v4si)_mm_setzero_si128()); @@ -318,7 +318,7 @@ _mm256_mask_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { +_mm256_maskz_dpwusd_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) { return (__m256i)__builtin_ia32_selectd_256( (__mmask8)__U, (__v8si)_mm256_dpwusd_epi32(__A, __B, __C), (__v8si)_mm256_setzero_si256()); @@ -331,7 +331,7 @@ _mm_mask_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { +_mm_maskz_dpwusds_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) { return (__m128i)__builtin_ia32_selectd_128( (__mmask8)__U, (__v4si)_mm_dpwusds_epi32(__A, __B, __C), (__v4si)_mm_setzero_si128()); @@ -344,7 +344,7 @@ _mm256_mask_dpwusds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { } static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwusds_epi32( - __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { + __mmask8 __U, __m256i __A, __m256i __B, __m256i __C) { return (__m256i)__builtin_ia32_selectd_256( (__mmask8)__U, (__v8si)_mm256_dpwusds_epi32(__A, __B, __C), (__v8si)_mm256_setzero_si256()); @@ -357,7 +357,7 @@ _mm_mask_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { +_mm_maskz_dpwuud_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) { return (__m128i)__builtin_ia32_selectd_128( (__mmask8)__U, (__v4si)_mm_dpwuud_epi32(__A, __B, __C), (__v4si)_mm_setzero_si128()); @@ -370,7 +370,7 @@ _mm256_mask_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { +_mm256_maskz_dpwuud_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) { return (__m256i)__builtin_ia32_selectd_256( (__mmask8)__U, (__v8si)_mm256_dpwuud_epi32(__A, __B, __C), (__v8si)_mm256_setzero_si256()); @@ -383,7 +383,7 @@ _mm_mask_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) { +_mm_maskz_dpwuuds_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) { return (__m128i)__builtin_ia32_selectd_128( (__mmask8)__U, (__v4si)_mm_dpwuuds_epi32(__A, __B, __C), (__v4si)_mm_setzero_si128()); @@ -396,7 +396,7 @@ _mm256_mask_dpwuuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { } static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwuuds_epi32( - __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) { + __mmask8 __U, __m256i __A, __m256i __B, __m256i __C) { return (__m256i)__builtin_ia32_selectd_256( (__mmask8)__U, (__v8si)_mm256_dpwuuds_epi32(__A, __B, __C), (__v8si)_mm256_setzero_si256()); diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h index 2b7f5043e09e..6206a347852b 100644 --- a/clang/lib/Headers/opencl-c-base.h +++ b/clang/lib/Headers/opencl-c-base.h @@ -697,7 +697,16 @@ template <typename _Tp> struct __remove_address_space<__constant _Tp> { #if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ >= CL_VERSION_1_2) // OpenCL v1.2 s6.12.13, v2.0 s6.13.13 - printf -int printf(__constant const char* st, ...) __attribute__((format(printf, 1, 2))); +#ifdef __OPENCL_CPP_VERSION__ +#define CLINKAGE extern "C" +#else +#define CLINKAGE +#endif + +CLINKAGE int printf(__constant const char *st, ...) + __attribute__((format(printf, 1, 2))); + +#undef CLINKAGE #endif #ifdef cl_intel_device_side_avc_motion_estimation diff --git a/clang/lib/Headers/ptrauth.h b/clang/lib/Headers/ptrauth.h index 7f7d387cbdfd..f902ca1e3bbd 100644 --- a/clang/lib/Headers/ptrauth.h +++ b/clang/lib/Headers/ptrauth.h @@ -95,7 +95,7 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t; __ptrauth qualifier; the compiler will perform this check automatically. */ -#if __has_feature(ptrauth_intrinsics) +#if __has_feature(ptrauth_intrinsics) || defined(__PTRAUTH__) /* Strip the signature from a value without authenticating it. @@ -388,6 +388,6 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t; #define __ptrauth_objc_isa_uintptr #define __ptrauth_objc_super_pointer -#endif /* __has_feature(ptrauth_intrinsics) */ +#endif /* __has_feature(ptrauth_intrinsics) || defined(__PTRAUTH__) */ #endif /* __PTRAUTH_H */ diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 1f695b4a8676..b282a600c0e5 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -174,8 +174,6 @@ void Lexer::InitLexer(const char *BufStart, const char *BufPtr, ExtendedTokenMode = 0; NewLinePtr = nullptr; - - IsFirstPPToken = true; } /// Lexer constructor - Create a new lexer object for the specified buffer @@ -3225,7 +3223,6 @@ std::optional<Token> Lexer::peekNextPPToken() { bool atStartOfLine = IsAtStartOfLine; bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; bool leadingSpace = HasLeadingSpace; - bool isFirstPPToken = IsFirstPPToken; Token Tok; Lex(Tok); @@ -3236,7 +3233,6 @@ std::optional<Token> Lexer::peekNextPPToken() { HasLeadingSpace = leadingSpace; IsAtStartOfLine = atStartOfLine; IsAtPhysicalStartOfLine = atPhysicalStartOfLine; - IsFirstPPToken = isFirstPPToken; // Restore the lexer back to non-skipping mode. LexingRawMode = false; @@ -3726,11 +3722,6 @@ bool Lexer::Lex(Token &Result) { HasLeadingEmptyMacro = false; } - if (IsFirstPPToken) { - Result.setFlag(Token::FirstPPToken); - IsFirstPPToken = false; - } - bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; IsAtPhysicalStartOfLine = false; bool isRawLex = isLexingRawMode(); diff --git a/clang/lib/Lex/LiteralSupport.cpp b/clang/lib/Lex/LiteralSupport.cpp index a62508e3e27b..5b08d7f0efe5 100644 --- a/clang/lib/Lex/LiteralSupport.cpp +++ b/clang/lib/Lex/LiteralSupport.cpp @@ -1467,7 +1467,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) { if (s != PossibleNewDigitStart) DigitsBegin = PossibleNewDigitStart; else - IsSingleZero = (s == ThisTokEnd); // Is the only thing we've seen a 0? + IsSingleZero = (s == ThisTokBegin + 1); if (s == ThisTokEnd) return; // Done, simple octal number like 01234 diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index bcd3ea60ce3d..2120e45dd8f8 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -43,6 +43,7 @@ #include "clang/Lex/MacroArgs.h" #include "clang/Lex/MacroInfo.h" #include "clang/Lex/ModuleLoader.h" +#include "clang/Lex/NoTrivialPPDirectiveTracer.h" #include "clang/Lex/Pragma.h" #include "clang/Lex/PreprocessingRecord.h" #include "clang/Lex/PreprocessorLexer.h" @@ -247,8 +248,6 @@ void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { llvm::errs() << " [LeadingSpace]"; if (Tok.isExpandDisabled()) llvm::errs() << " [ExpandDisabled]"; - if (Tok.isFirstPPToken()) - llvm::errs() << " [First pp-token]"; if (Tok.needsCleaning()) { const char *Start = SourceMgr.getCharacterData(Tok.getLocation()); llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) @@ -577,8 +576,11 @@ void Preprocessor::EnterMainSourceFile() { // export module M; // error: module declaration must occur // // at the start of the translation unit. if (getLangOpts().CPlusPlusModules) { + auto Tracer = std::make_unique<NoTrivialPPDirectiveTracer>(*this); + DirTracer = Tracer.get(); + addPPCallbacks(std::move(Tracer)); std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken(); - if (FirstPPTok && FirstPPTok->isFirstPPToken()) + if (FirstPPTok) FirstPPTokenLoc = FirstPPTok->getLocation(); } } @@ -940,6 +942,8 @@ void Preprocessor::Lex(Token &Result) { StdCXXImportSeqState.handleHeaderName(); break; case tok::kw_export: + if (hasSeenNoTrivialPPDirective()) + Result.setFlag(Token::HasSeenNoTrivialPPDirective); TrackGMFState.handleExport(); StdCXXImportSeqState.handleExport(); ModuleDeclState.handleExport(); @@ -966,6 +970,8 @@ void Preprocessor::Lex(Token &Result) { } break; } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) { + if (hasSeenNoTrivialPPDirective()) + Result.setFlag(Token::HasSeenNoTrivialPPDirective); TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq()); ModuleDeclState.handleModule(); break; @@ -1680,3 +1686,31 @@ const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const { return nullptr; } + +bool Preprocessor::hasSeenNoTrivialPPDirective() const { + return DirTracer && DirTracer->hasSeenNoTrivialPPDirective(); +} + +bool NoTrivialPPDirectiveTracer::hasSeenNoTrivialPPDirective() const { + return SeenNoTrivialPPDirective; +} + +void NoTrivialPPDirectiveTracer::setSeenNoTrivialPPDirective() { + if (InMainFile && !SeenNoTrivialPPDirective) + SeenNoTrivialPPDirective = true; +} + +void NoTrivialPPDirectiveTracer::LexedFileChanged( + FileID FID, LexedFileChangeReason Reason, + SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) { + InMainFile = (FID == PP.getSourceManager().getMainFileID()); +} + +void NoTrivialPPDirectiveTracer::MacroExpands(const Token &MacroNameTok, + const MacroDefinition &MD, + SourceRange Range, + const MacroArgs *Args) { + // FIXME: Does only enable builtin macro expansion make sense? + if (!MD.getMacroInfo()->isBuiltinMacro()) + setSeenNoTrivialPPDirective(); +} diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index bc238a9517a3..3515343202de 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -3342,7 +3342,8 @@ ExprResult Parser::ParseBlockLiteralExpression() { Actions.ActOnBlockError(CaretLoc, getCurScope()); return ExprError(); } - + EnterExpressionEvaluationContextForFunction PotentiallyEvaluated( + Actions, Sema::ExpressionEvaluationContext::PotentiallyEvaluated); StmtResult Stmt(ParseCompoundStatementBody()); BlockScope.Exit(); if (!Stmt.isInvalid()) diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 8834bf80c401..447d0283431b 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -2361,9 +2361,10 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) { // Parse a global-module-fragment, if present. if (getLangOpts().CPlusPlusModules && Tok.is(tok::semi)) { SourceLocation SemiLoc = ConsumeToken(); - if (!Introducer.isFirstPPToken()) { + if (ImportState != Sema::ModuleImportState::FirstDecl || + Introducer.hasSeenNoTrivialPPDirective()) { Diag(StartLoc, diag::err_global_module_introducer_not_at_start) - << SourceRange(StartLoc, SemiLoc); + << SourceRange(StartLoc, SemiLoc); return nullptr; } if (MDK == Sema::ModuleDeclKind::Interface) { @@ -2418,7 +2419,8 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) { ExpectAndConsumeSemi(diag::err_module_expected_semi); return Actions.ActOnModuleDecl(StartLoc, ModuleLoc, MDK, Path, Partition, - ImportState, Introducer.isFirstPPToken()); + ImportState, + Introducer.hasSeenNoTrivialPPDirective()); } Decl *Parser::ParseModuleImport(SourceLocation AtLoc, diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp index 5e75c64eb2b9..ec39bca6039f 100644 --- a/clang/lib/Sema/AnalysisBasedWarnings.cpp +++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp @@ -503,8 +503,12 @@ static bool areAllValuesNoReturn(const VarDecl *VD, const CFGBlock &VarBlk, TransferFunctions TF(VD); BackwardDataflowWorklist Worklist(*AC.getCFG(), AC); + llvm::DenseSet<const CFGBlock *> Visited; Worklist.enqueueBlock(&VarBlk); while (const CFGBlock *B = Worklist.dequeue()) { + if (Visited.contains(B)) + continue; + Visited.insert(B); // First check the current block. for (CFGBlock::const_reverse_iterator ri = B->rbegin(), re = B->rend(); ri != re; ++ri) { @@ -2887,8 +2891,7 @@ void clang::sema::AnalysisBasedWarnings::IssueWarnings( .setAlwaysAdd(Stmt::UnaryOperatorClass); } - bool EnableLifetimeSafetyAnalysis = !Diags.isIgnored( - diag::warn_experimental_lifetime_safety_dummy_warning, D->getBeginLoc()); + bool EnableLifetimeSafetyAnalysis = S.getLangOpts().EnableLifetimeSafety; // Install the logical handler. std::optional<LogicalErrorHandler> LEH; if (LogicalErrorHandler::hasActiveDiagnostics(Diags, D->getBeginLoc())) { diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index dd5b710d7e1d..41bcf8fd493f 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -12014,13 +12014,20 @@ static void DiagnoseMixedUnicodeImplicitConversion(Sema &S, const Type *Source, SourceLocation CC) { assert(Source->isUnicodeCharacterType() && Target->isUnicodeCharacterType() && Source != Target); + + // Lone surrogates have a distinct representation in UTF-32. + // Converting between UTF-16 and UTF-32 codepoints seems very widespread, + // so don't warn on such conversion. + if (Source->isChar16Type() && Target->isChar32Type()) + return; + Expr::EvalResult Result; if (E->EvaluateAsInt(Result, S.getASTContext(), Expr::SE_AllowSideEffects, S.isConstantEvaluatedContext())) { llvm::APSInt Value(32); Value = Result.Val.getInt(); bool IsASCII = Value <= 0x7F; - bool IsBMP = Value <= 0xD7FF || (Value >= 0xE000 && Value <= 0xFFFF); + bool IsBMP = Value <= 0xDFFF || (Value >= 0xE000 && Value <= 0xFFFF); bool ConversionPreservesSemantics = IsASCII || (!Source->isChar8Type() && !Target->isChar8Type() && IsBMP); diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp index 834417f8e15a..20567a6d9d1a 100644 --- a/clang/lib/Sema/SemaConcept.cpp +++ b/clang/lib/Sema/SemaConcept.cpp @@ -1097,10 +1097,6 @@ static bool CheckFunctionConstraintsWithoutInstantiation( } Sema::ContextRAII SavedContext(SemaRef, FD); - std::optional<Sema::CXXThisScopeRAII> ThisScope; - if (auto *Method = dyn_cast<CXXMethodDecl>(FD)) - ThisScope.emplace(SemaRef, /*Record=*/Method->getParent(), - /*ThisQuals=*/Method->getMethodQualifiers()); return SemaRef.CheckConstraintSatisfaction( Template, TemplateAC, MLTAL, PointOfInstantiation, Satisfaction); } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 14403e65e8f4..d43dc2297c6f 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -3267,6 +3267,14 @@ void Sema::mergeDeclAttributes(NamedDecl *New, Decl *Old, if (isa<UsedAttr>(I) || isa<RetainAttr>(I)) continue; + if (isa<InferredNoReturnAttr>(I)) { + if (auto *FD = dyn_cast<FunctionDecl>(New)) { + if (FD->getTemplateSpecializationKind() == TSK_ExplicitSpecialization) + continue; // Don't propagate inferred noreturn attributes to explicit + // specializations. + } + } + if (mergeDeclAttribute(*this, New, I, LocalAMK)) foundAny = true; } @@ -19643,6 +19651,7 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl, Q && Q.isAddressDiscriminated()) { Record->setArgPassingRestrictions( RecordArgPassingKind::CanNeverPassInRegs); + Record->setNonTrivialToPrimitiveCopy(true); } } diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 5f481ed1f713..a7897bdfe6e0 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -1970,6 +1970,13 @@ void clang::inferNoReturnAttr(Sema &S, const Decl *D) { if (!FD) return; + // Skip explicit specializations here as they may have + // a user-provided definition that may deliberately differ from the primary + // template. If an explicit specialization truly never returns, the user + // should explicitly mark it with [[noreturn]]. + if (FD->getTemplateSpecializationKind() == TSK_ExplicitSpecialization) + return; + auto *NonConstFD = const_cast<FunctionDecl *>(FD); DiagnosticsEngine &Diags = S.getDiagnostics(); if (Diags.isIgnored(diag::warn_falloff_nonvoid, FD->getLocation()) && @@ -5011,10 +5018,10 @@ void Sema::AddModeAttr(Decl *D, const AttributeCommonInfo &CI, static void handleNonStringAttr(Sema &S, Decl *D, const ParsedAttr &AL) { // This only applies to fields and variable declarations which have an array - // type. + // type or pointer type, with character elements. QualType QT = cast<ValueDecl>(D)->getType(); - if (!QT->isArrayType() || - !QT->getBaseElementTypeUnsafe()->isAnyCharacterType()) { + if ((!QT->isArrayType() && !QT->isPointerType()) || + !QT->getPointeeOrArrayElementType()->isAnyCharacterType()) { S.Diag(D->getBeginLoc(), diag::warn_attribute_non_character_array) << AL << AL.isRegularKeywordAttribute() << QT << AL.getRange(); return; diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 728ada33e2e6..5421e9562c8b 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -9309,14 +9309,14 @@ AssignConvertType Sema::CheckAssignmentConstraints(QualType LHSType, // If we have an atomic type, try a non-atomic assignment, then just add an // atomic qualification step. if (const AtomicType *AtomicTy = dyn_cast<AtomicType>(LHSType)) { - AssignConvertType result = + AssignConvertType Result = CheckAssignmentConstraints(AtomicTy->getValueType(), RHS, Kind); - if (result != AssignConvertType::Compatible) - return result; + if (!IsAssignConvertCompatible(Result)) + return Result; if (Kind != CK_NoOp && ConvertRHS) RHS = ImpCastExprToType(RHS.get(), AtomicTy->getValueType(), Kind); Kind = CK_NonAtomicToAtomic; - return AssignConvertType::Compatible; + return Result; } // If the left-hand side is a reference type, then we are in a diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index 25afa2f4dfe7..a08971c6526f 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -347,6 +347,8 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II, CheckTypenameType(ElaboratedTypeKeyword::None, SourceLocation(), SS.getWithLocInContext(Context), II, NameLoc, &TSI, /*DeducedTSTContext=*/true); + if (T.isNull()) + return ParsedType(); return CreateParsedType(T, TSI); } diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index 7c982bcd63d7..10a390286ec7 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -264,10 +264,11 @@ Sema::DeclGroupPtrTy Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc, ModuleDeclKind MDK, ModuleIdPath Path, ModuleIdPath Partition, ModuleImportState &ImportState, - bool IntroducerIsFirstPPToken) { + bool SeenNoTrivialPPDirective) { assert(getLangOpts().CPlusPlusModules && "should only have module decl in standard C++ modules"); + bool IsFirstDecl = ImportState == ModuleImportState::FirstDecl; bool SeenGMF = ImportState == ModuleImportState::GlobalFragment; // If any of the steps here fail, we count that as invalidating C++20 // module state; @@ -335,7 +336,8 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc, // In C++20, A module directive may only appear as the first preprocessing // tokens in a file (excluding the global module fragment.). - if (getLangOpts().CPlusPlusModules && !IntroducerIsFirstPPToken && !SeenGMF) { + if (getLangOpts().CPlusPlusModules && + (!IsFirstDecl || SeenNoTrivialPPDirective) && !SeenGMF) { Diag(ModuleLoc, diag::err_module_decl_not_at_start); SourceLocation BeginLoc = PP.getMainFileFirstPPTokenLoc(); Diag(BeginLoc, diag::note_global_module_introducer_missing) diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index f3baf0c3ef3b..1b54628c5e56 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -245,7 +245,6 @@ void StandardConversionSequence::setAsIdentityConversion() { IsLvalueReference = true; BindsToFunctionLvalue = false; BindsToRvalue = false; - IsImplicitObjectArgumentQualificationConversion = false; BindsImplicitObjectArgumentWithoutRefQualifier = false; ObjCLifetimeConversionBinding = false; FromBracedInitList = false; @@ -5318,7 +5317,6 @@ TryReferenceInit(Sema &S, Expr *Init, QualType DeclType, ICS.Standard.DirectBinding = BindsDirectly; ICS.Standard.IsLvalueReference = !isRValRef; ICS.Standard.BindsToFunctionLvalue = T2->isFunctionType(); - ICS.Standard.IsImplicitObjectArgumentQualificationConversion = false; ICS.Standard.BindsToRvalue = InitCategory.isRValue(); ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false; ICS.Standard.ObjCLifetimeConversionBinding = @@ -5498,7 +5496,6 @@ TryReferenceInit(Sema &S, Expr *Init, QualType DeclType, ICS.Standard.IsLvalueReference = !isRValRef; ICS.Standard.BindsToFunctionLvalue = false; ICS.Standard.BindsToRvalue = true; - ICS.Standard.IsImplicitObjectArgumentQualificationConversion = false; ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false; ICS.Standard.ObjCLifetimeConversionBinding = false; } else if (ICS.isUserDefined()) { @@ -5521,8 +5518,6 @@ TryReferenceInit(Sema &S, Expr *Init, QualType DeclType, ICS.UserDefined.After.IsLvalueReference = !isRValRef; ICS.UserDefined.After.BindsToFunctionLvalue = false; ICS.UserDefined.After.BindsToRvalue = !LValRefType; - ICS.UserDefined.After.IsImplicitObjectArgumentQualificationConversion = - false; ICS.UserDefined.After.BindsImplicitObjectArgumentWithoutRefQualifier = false; ICS.UserDefined.After.ObjCLifetimeConversionBinding = false; ICS.UserDefined.After.FromBracedInitList = false; @@ -5807,7 +5802,6 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType, StandardConversionSequence &SCS = Result.isStandard() ? Result.Standard : Result.UserDefined.After; SCS.ReferenceBinding = true; - SCS.IsImplicitObjectArgumentQualificationConversion = false; SCS.IsLvalueReference = ToType->isLValueReferenceType(); SCS.BindsToRvalue = true; SCS.BindsToFunctionLvalue = false; @@ -6005,12 +5999,8 @@ static ImplicitConversionSequence TryObjectArgumentInitialization( // affects the conversion rank. QualType ClassTypeCanon = S.Context.getCanonicalType(ClassType); ImplicitConversionKind SecondKind; - bool IsQualificationConversion = false; - if (ImplicitParamType.getCanonicalType() == FromTypeCanon) { + if (ClassTypeCanon == FromTypeCanon.getLocalUnqualifiedType()) { SecondKind = ICK_Identity; - } else if (ClassTypeCanon == FromTypeCanon.getLocalUnqualifiedType()) { - SecondKind = ICK_Identity; - IsQualificationConversion = true; } else if (S.IsDerivedFrom(Loc, FromType, ClassType)) { SecondKind = ICK_Derived_To_Base; } else if (!Method->isExplicitObjectMemberFunction()) { @@ -6051,8 +6041,6 @@ static ImplicitConversionSequence TryObjectArgumentInitialization( ICS.Standard.setFromType(FromType); ICS.Standard.setAllToTypes(ImplicitParamType); ICS.Standard.ReferenceBinding = true; - ICS.Standard.IsImplicitObjectArgumentQualificationConversion = - IsQualificationConversion; ICS.Standard.DirectBinding = true; ICS.Standard.IsLvalueReference = Method->getRefQualifier() != RQ_RValue; ICS.Standard.BindsToFunctionLvalue = false; diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp index f85826aecadf..f46be75bda20 100644 --- a/clang/lib/Sema/SemaStmt.cpp +++ b/clang/lib/Sema/SemaStmt.cpp @@ -2287,7 +2287,11 @@ StmtResult Sema::ActOnForStmt(SourceLocation ForLoc, SourceLocation LParenLoc, // we can diagnose if we don't see any variable declarations. This // covers a case like declaring a typedef, function, or structure // type rather than a variable. - NonVarSeen = DI; + // + // Note, _Static_assert is acceptable because it does not declare an + // identifier at all, so "for object having" does not apply. + if (!isa<StaticAssertDecl>(DI)) + NonVarSeen = DI; } } // Diagnose if we saw a non-variable declaration but no variable diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index b76619fc5026..f067aedf1cc9 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -4749,8 +4749,6 @@ Sema::CheckConceptTemplateId(const CXXScopeSpec &SS, EnterExpressionEvaluationContext EECtx{ *this, ExpressionEvaluationContext::Unevaluated, CSD}; - ContextRAII CurContext(*this, CSD->getDeclContext(), - /*NewThisContext=*/false); if (!AreArgsDependent && CheckConstraintSatisfaction( NamedConcept, AssociatedConstraint(NamedConcept->getConstraintExpr()), diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp index d09a72b71b80..ce78ecc2d4a2 100644 --- a/clang/lib/Sema/SemaTemplateDeduction.cpp +++ b/clang/lib/Sema/SemaTemplateDeduction.cpp @@ -5525,6 +5525,15 @@ static TemplateDeductionResult CheckDeductionConsistency( // FIXME: A substitution can be incomplete on a non-structural part of the // type. Use the canonical type for now, until the TemplateInstantiator can // deal with that. + + // Workaround: Implicit deduction guides use InjectedClassNameTypes, whereas + // the explicit guides don't. The substitution doesn't transform these types, + // so let it transform their specializations instead. + bool IsDeductionGuide = isa<CXXDeductionGuideDecl>(FTD->getTemplatedDecl()); + if (IsDeductionGuide) { + if (auto *Injected = P->getAs<InjectedClassNameType>()) + P = Injected->getInjectedSpecializationType(); + } QualType InstP = S.SubstType(P.getCanonicalType(), MLTAL, FTD->getLocation(), FTD->getDeclName(), &IsIncompleteSubstitution); if (InstP.isNull() && !IsIncompleteSubstitution) @@ -5539,9 +5548,15 @@ static TemplateDeductionResult CheckDeductionConsistency( if (auto *PA = dyn_cast<PackExpansionType>(A); PA && !isa<PackExpansionType>(InstP)) A = PA->getPattern(); - if (!S.Context.hasSameType( - S.Context.getUnqualifiedArrayType(InstP.getNonReferenceType()), - S.Context.getUnqualifiedArrayType(A.getNonReferenceType()))) + auto T1 = S.Context.getUnqualifiedArrayType(InstP.getNonReferenceType()); + auto T2 = S.Context.getUnqualifiedArrayType(A.getNonReferenceType()); + if (IsDeductionGuide) { + if (auto *Injected = T1->getAs<InjectedClassNameType>()) + T1 = Injected->getInjectedSpecializationType(); + if (auto *Injected = T2->getAs<InjectedClassNameType>()) + T2 = Injected->getInjectedSpecializationType(); + } + if (!S.Context.hasSameType(T1, T2)) return TemplateDeductionResult::NonDeducedMismatch; return TemplateDeductionResult::Success; } diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp index e2c3cdcd536b..d2b87f2702a9 100644 --- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp @@ -5685,7 +5685,7 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation, }; Function->setDeclarationNameLoc(NameLocPointsToPattern()); - EnterExpressionEvaluationContext EvalContext( + EnterExpressionEvaluationContextForFunction EvalContext( *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated); Qualifiers ThisTypeQuals; diff --git a/clang/lib/Sema/SemaTypeTraits.cpp b/clang/lib/Sema/SemaTypeTraits.cpp index 1d8687e4bf1c..21ba19ee551c 100644 --- a/clang/lib/Sema/SemaTypeTraits.cpp +++ b/clang/lib/Sema/SemaTypeTraits.cpp @@ -1768,7 +1768,10 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, // Objective-C lifetime, this is a non-trivial assignment. if (LhsT.getNonReferenceType().hasNonTrivialObjCLifetime()) return false; - + ASTContext &Context = Self.getASTContext(); + if (Context.containsAddressDiscriminatedPointerAuth(LhsT) || + Context.containsAddressDiscriminatedPointerAuth(RhsT)) + return false; return !Result.get()->hasNonTrivialCall(Self.Context); } diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 30e097314959..3e7ccfdd9db5 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -3103,6 +3103,10 @@ ASTReader::ReadControlBlock(ModuleFile &F, F.Kind == MK_ImplicitModule) N = ForceValidateUserInputs ? NumUserInputs : 0; + if (N != 0) + Diag(diag::remark_module_validation) + << N << F.ModuleName << F.FileName; + for (unsigned I = 0; I < N; ++I) { InputFile IF = getInputFile(F, I+1, Complain); if (!IF.getFile() || IF.isOutOfDate()) diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 1a20fc9595dc..3caf9ebb3218 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -4340,8 +4340,7 @@ private: // parent of parent. We DON'T remove the enum constant from its parent. So // we don't need to care about merging problems here. if (auto *ECD = dyn_cast<EnumConstantDecl>(D); - ECD && DC.isFileContext() && ECD->getOwningModule() && - ECD->getTopLevelOwningNamedModule()->isNamedModule()) { + ECD && DC.isFileContext() && ECD->getTopLevelOwningNamedModule()) { if (llvm::all_of( DC.noload_lookup( cast<EnumDecl>(ECD->getDeclContext())->getDeclName()), diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp index f42bdc16d815..88ad8dd6495d 100644 --- a/clang/lib/Serialization/ModuleCache.cpp +++ b/clang/lib/Serialization/ModuleCache.cpp @@ -34,8 +34,10 @@ public: } std::time_t getModuleTimestamp(StringRef ModuleFilename) override { + std::string TimestampFilename = + serialization::ModuleFile::getTimestampFilename(ModuleFilename); llvm::sys::fs::file_status Status; - if (llvm::sys::fs::status(ModuleFilename, Status) != std::error_code{}) + if (llvm::sys::fs::status(TimestampFilename, Status) != std::error_code{}) return 0; return llvm::sys::toTimeT(Status.getLastModificationTime()); } diff --git a/clang/lib/StaticAnalyzer/Checkers/AssumeModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/AssumeModeling.cpp index 1e3adb4f266c..789c7772d123 100644 --- a/clang/lib/StaticAnalyzer/Checkers/AssumeModeling.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/AssumeModeling.cpp @@ -45,7 +45,6 @@ void AssumeModelingChecker::checkPostStmt(const AttributedStmt *A, continue; const auto *Assumption = AssumptionVal.getAsInteger(); - assert(Assumption && "We should know the exact outcome of an assume expr"); if (Assumption && Assumption->isZero()) { C.addSink(); } diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp index fa8e669b6bb2..ab29f86cec32 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp @@ -916,7 +916,8 @@ VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *Ex, QualType T = Ex->getTypeOfArgument(); for (ExplodedNode *N : CheckedSet) { - if (Ex->getKind() == UETT_SizeOf) { + if (Ex->getKind() == UETT_SizeOf || Ex->getKind() == UETT_DataSizeOf || + Ex->getKind() == UETT_CountOf) { if (!T->isIncompleteType() && !T->isConstantSizeType()) { assert(T->isVariableArrayType() && "Unknown non-constant-sized type."); diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp index 85353848aa12..dc715c7d46d8 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp @@ -71,21 +71,30 @@ void ExprEngine::performTrivialCopy(NodeBuilder &Bldr, ExplodedNode *Pred, Bldr.takeNodes(Pred); assert(ThisRD); - SVal V = Call.getArgSVal(0); - const Expr *VExpr = Call.getArgExpr(0); - // If the value being copied is not unknown, load from its location to get - // an aggregate rvalue. - if (std::optional<Loc> L = V.getAs<Loc>()) - V = Pred->getState()->getSVal(*L); - else - assert(V.isUnknownOrUndef()); + if (!ThisRD->isEmpty()) { + SVal V = Call.getArgSVal(0); + const Expr *VExpr = Call.getArgExpr(0); - ExplodedNodeSet Tmp; - evalLocation(Tmp, CallExpr, VExpr, Pred, Pred->getState(), V, - /*isLoad=*/true); - for (ExplodedNode *N : Tmp) - evalBind(Dst, CallExpr, N, ThisVal, V, true); + // If the value being copied is not unknown, load from its location to get + // an aggregate rvalue. + if (std::optional<Loc> L = V.getAs<Loc>()) + V = Pred->getState()->getSVal(*L); + else + assert(V.isUnknownOrUndef()); + + ExplodedNodeSet Tmp; + evalLocation(Tmp, CallExpr, VExpr, Pred, Pred->getState(), V, + /*isLoad=*/true); + for (ExplodedNode *N : Tmp) + evalBind(Dst, CallExpr, N, ThisVal, V, true); + } else { + // We can't copy empty classes because of empty base class optimization. + // In that case, copying the empty base class subobject would overwrite the + // object that it overlaps with - so let's not do that. + // See issue-157467.cpp for an example. + Dst.Add(Pred); + } PostStmt PS(CallExpr, LCtx); for (ExplodedNode *N : Dst) { diff --git a/compiler-rt/lib/builtins/aarch64/lse.S b/compiler-rt/lib/builtins/aarch64/lse.S index d7c1db7243ef..a444d82892c3 100644 --- a/compiler-rt/lib/builtins/aarch64/lse.S +++ b/compiler-rt/lib/builtins/aarch64/lse.S @@ -264,7 +264,7 @@ END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM)) NO_EXEC_STACK_DIRECTIVE -// GNU property note for BTI and PAC -GNU_PROPERTY_BTI_PAC +// GNU property note for BTI, PAC, and GCS +GNU_PROPERTY_BTI_PAC_GCS #endif // defined(__aarch64__) || defined(__arm64ec__) diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi.S b/compiler-rt/lib/builtins/aarch64/sme-abi.S index 7c47336cfc57..d5510ac0cfa5 100644 --- a/compiler-rt/lib/builtins/aarch64/sme-abi.S +++ b/compiler-rt/lib/builtins/aarch64/sme-abi.S @@ -371,5 +371,5 @@ END_COMPILERRT_FUNCTION(__arm_sme_restore) NO_EXEC_STACK_DIRECTIVE -// GNU property note for BTI and PAC -GNU_PROPERTY_BTI_PAC +// GNU property note for BTI, PAC, and GCS +GNU_PROPERTY_BTI_PAC_GCS diff --git a/compiler-rt/lib/builtins/assembly.h b/compiler-rt/lib/builtins/assembly.h index 89372f18c84b..d7db7d818945 100644 --- a/compiler-rt/lib/builtins/assembly.h +++ b/compiler-rt/lib/builtins/assembly.h @@ -79,11 +79,12 @@ #define FUNC_ALIGN #endif -// BTI and PAC gnu property note +// BTI, PAC, and GCS gnu property note #define NT_GNU_PROPERTY_TYPE_0 5 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2 +#define GNU_PROPERTY_AARCH64_FEATURE_1_GCS 4 #if defined(__ARM_FEATURE_BTI_DEFAULT) #define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI @@ -97,6 +98,12 @@ #define PAC_FLAG 0 #endif +#if defined(__ARM_FEATURE_GCS_DEFAULT) +#define GCS_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_GCS +#else +#define GCS_FLAG 0 +#endif + #define GNU_PROPERTY(type, value) \ .pushsection .note.gnu.property, "a" SEPARATOR \ .p2align 3 SEPARATOR \ @@ -118,11 +125,12 @@ #define BTI_J #endif -#if (BTI_FLAG | PAC_FLAG) != 0 -#define GNU_PROPERTY_BTI_PAC \ - GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG) +#if (BTI_FLAG | PAC_FLAG | GCS_FLAG) != 0 +#define GNU_PROPERTY_BTI_PAC_GCS \ + GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, \ + BTI_FLAG | PAC_FLAG | GCS_FLAG) #else -#define GNU_PROPERTY_BTI_PAC +#define GNU_PROPERTY_BTI_PAC_GCS #endif #if defined(__clang__) || defined(__GCC_HAVE_DWARF2_CFI_ASM) diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c index be002dd71992..d7880529ebe7 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64.c +++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c @@ -34,12 +34,12 @@ typedef struct __ifunc_arg_t { _Bool __aarch64_have_lse_atomics __attribute__((visibility("hidden"), nocommon)) = false; -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__OpenBSD__) // clang-format off: should not reorder sys/auxv.h alphabetically #include <sys/auxv.h> // clang-format on #include "aarch64/hwcap.inc" -#include "aarch64/lse_atomics/freebsd.inc" +#include "aarch64/lse_atomics/elf_aux_info.inc" #elif defined(__Fuchsia__) #include "aarch64/hwcap.inc" #include "aarch64/lse_atomics/fuchsia.inc" @@ -68,9 +68,9 @@ struct { // clang-format off #if defined(__APPLE__) #include "aarch64/fmv/apple.inc" -#elif defined(__FreeBSD__) +#elif defined(__FreeBSD__) || defined(__OpenBSD__) #include "aarch64/fmv/mrs.inc" -#include "aarch64/fmv/freebsd.inc" +#include "aarch64/fmv/elf_aux_info.inc" #elif defined(__Fuchsia__) #include "aarch64/fmv/fuchsia.inc" #elif defined(__ANDROID__) diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/freebsd.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/elf_aux_info.inc index aa975dc854f9..aa975dc854f9 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/freebsd.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/elf_aux_info.inc diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/freebsd.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/elf_aux_info.inc index 4a1f9c2c27c8..4a1f9c2c27c8 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/freebsd.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/elf_aux_info.inc diff --git a/compiler-rt/lib/builtins/crtbegin.c b/compiler-rt/lib/builtins/crtbegin.c index d5f7756308b0..447474bd0b69 100644 --- a/compiler-rt/lib/builtins/crtbegin.c +++ b/compiler-rt/lib/builtins/crtbegin.c @@ -54,22 +54,33 @@ static void __attribute__((used)) __do_init(void) { } #ifdef CRT_HAS_INITFINI_ARRAY -#if __has_feature(ptrauth_init_fini) +# if __has_feature(ptrauth_init_fini) // TODO: use __ptrauth-qualified pointers when they are supported on clang side -#if __has_feature(ptrauth_init_fini_address_discrimination) +# if __has_feature(ptrauth_init_fini_address_discrimination) __attribute__((section(".init_array"), used)) static void *__init = ptrauth_sign_constant(&__do_init, ptrauth_key_init_fini_pointer, ptrauth_blend_discriminator( &__init, __ptrauth_init_fini_discriminator)); -#else +# else __attribute__((section(".init_array"), used)) static void *__init = ptrauth_sign_constant(&__do_init, ptrauth_key_init_fini_pointer, __ptrauth_init_fini_discriminator); -#endif -#else +# endif +# elif __has_feature(ptrauth_calls) +# ifdef __aarch64__ +// If ptrauth_init_fini feature is not present, compiler emits raw unsigned +// pointers in .init_array. Use inline assembly to avoid implicit signing of +// __do_init function pointer with ptrauth_calls enabled. +__asm__(".pushsection .init_array,\"aw\",@init_array\n\t" + ".xword __do_init\n\t" + ".popsection"); +# else +# error "ptrauth_calls is only supported for AArch64" +# endif +# else __attribute__((section(".init_array"), used)) static void (*__init)(void) = __do_init; -#endif +# endif #elif defined(__i386__) || defined(__x86_64__) __asm__(".pushsection .init,\"ax\",@progbits\n\t" "call __do_init\n\t" @@ -125,22 +136,33 @@ static void __attribute__((used)) __do_fini(void) { } #ifdef CRT_HAS_INITFINI_ARRAY -#if __has_feature(ptrauth_init_fini) +# if __has_feature(ptrauth_init_fini) // TODO: use __ptrauth-qualified pointers when they are supported on clang side -#if __has_feature(ptrauth_init_fini_address_discrimination) +# if __has_feature(ptrauth_init_fini_address_discrimination) __attribute__((section(".fini_array"), used)) static void *__fini = ptrauth_sign_constant(&__do_fini, ptrauth_key_init_fini_pointer, ptrauth_blend_discriminator( &__fini, __ptrauth_init_fini_discriminator)); -#else +# else __attribute__((section(".fini_array"), used)) static void *__fini = ptrauth_sign_constant(&__do_fini, ptrauth_key_init_fini_pointer, __ptrauth_init_fini_discriminator); -#endif -#else +# endif +# elif __has_feature(ptrauth_calls) +# ifdef __aarch64__ +// If ptrauth_init_fini feature is not present, compiler emits raw unsigned +// pointers in .fini_array. Use inline assembly to avoid implicit signing of +// __do_fini function pointer with ptrauth_calls enabled. +__asm__(".pushsection .fini_array,\"aw\",@fini_array\n\t" + ".xword __do_fini\n\t" + ".popsection"); +# else +# error "ptrauth_calls is only supported for AArch64" +# endif +# else __attribute__((section(".fini_array"), used)) static void (*__fini)(void) = __do_fini; -#endif +# endif #elif defined(__i386__) || defined(__x86_64__) __asm__(".pushsection .fini,\"ax\",@progbits\n\t" "call __do_fini\n\t" diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S index fd20825e3dac..825f41156509 100644 --- a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S +++ b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S @@ -11,4 +11,4 @@ NO_EXEC_STACK_DIRECTIVE -GNU_PROPERTY_BTI_PAC +GNU_PROPERTY_BTI_PAC_GCS diff --git a/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S b/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S index 0c0abb6de861..b8d98b09ada2 100644 --- a/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S +++ b/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S @@ -99,4 +99,4 @@ ASM_TRAMPOLINE_ALIAS(_setjmp, setjmp) // We do not need executable stack. NO_EXEC_STACK_DIRECTIVE -GNU_PROPERTY_BTI_PAC +GNU_PROPERTY_BTI_PAC_GCS diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S index fd060c51cd8e..be82475101c8 100644 --- a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S +++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S @@ -157,4 +157,4 @@ mismatch: // We do not need executable stack. NO_EXEC_STACK_DIRECTIVE -GNU_PROPERTY_BTI_PAC +GNU_PROPERTY_BTI_PAC_GCS diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc index 08c2be47f535..673f284b6a04 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc @@ -344,12 +344,16 @@ static void ioctl_table_fill() { _(SOUND_PCM_WRITE_CHANNELS, WRITE, sizeof(int)); _(SOUND_PCM_WRITE_FILTER, WRITE, sizeof(int)); _(TCFLSH, NONE, 0); +# if SANITIZER_TERMIOS_IOCTL_CONSTANTS _(TCGETS, WRITE, struct_termios_sz); +# endif _(TCSBRK, NONE, 0); _(TCSBRKP, NONE, 0); +# if SANITIZER_TERMIOS_IOCTL_CONSTANTS _(TCSETS, READ, struct_termios_sz); _(TCSETSF, READ, struct_termios_sz); _(TCSETSW, READ, struct_termios_sz); +# endif _(TCXONC, NONE, 0); _(TIOCGLCKTRMIOS, WRITE, struct_termios_sz); _(TIOCGSOFTCAR, WRITE, sizeof(int)); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S index cdfa6f1d7f53..5066953980af 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S @@ -43,6 +43,6 @@ ASM_SIZE(vfork) ASM_INTERCEPTOR_TRAMPOLINE(vfork) ASM_TRAMPOLINE_ALIAS(vfork, vfork) -GNU_PROPERTY_BTI_PAC +GNU_PROPERTY_BTI_PAC_GCS #endif diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp index f5cb85bc1bf3..530ff90c4cd1 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -29,6 +29,7 @@ # include "sanitizer_solaris.h" # if SANITIZER_HAIKU +# define _GNU_SOURCE # define _DEFAULT_SOURCE # endif diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h index 196c0a988478..13099fe84b0a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -482,4 +482,19 @@ # define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 0 #endif +#if SANITIZER_LINUX +# if SANITIZER_GLIBC +// Workaround for +// glibc/commit/3d3572f59059e2b19b8541ea648a6172136ec42e +// Linux: Keep termios ioctl constants strictly internal +# if __GLIBC_PREREQ(2, 41) +# define SANITIZER_TERMIOS_IOCTL_CONSTANTS 0 +# else +# define SANITIZER_TERMIOS_IOCTL_CONSTANTS 1 +# endif +# else +# define SANITIZER_TERMIOS_IOCTL_CONSTANTS 1 +# endif +#endif + #endif // SANITIZER_PLATFORM_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp index 4c1e00528923..c4fa1e3c1f6f 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp @@ -71,14 +71,8 @@ #include <semaphore.h> #include <signal.h> #include <stddef.h> -#include <md5.h> -#include <sha224.h> -#include <sha256.h> -#include <sha384.h> -#include <sha512.h> #include <stdio.h> #include <stringlist.h> -#include <term.h> #include <termios.h> #include <time.h> #include <ttyent.h> @@ -370,22 +364,6 @@ const int si_SEGV_MAPERR = SEGV_MAPERR; const int si_SEGV_ACCERR = SEGV_ACCERR; const int unvis_valid = UNVIS_VALID; const int unvis_validpush = UNVIS_VALIDPUSH; - -const unsigned MD5_CTX_sz = sizeof(MD5_CTX); -const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH; - -#define SHA2_CONST(LEN) \ - const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX); \ - const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \ - const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH; \ - const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH - -SHA2_CONST(224); -SHA2_CONST(256); -SHA2_CONST(384); -SHA2_CONST(512); - -#undef SHA2_CONST } // namespace __sanitizer using namespace __sanitizer; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h index 382b67ce78eb..1cbb40e0b2ff 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -710,22 +710,6 @@ extern unsigned IOCTL_KDSKBMODE; extern const int si_SEGV_MAPERR; extern const int si_SEGV_ACCERR; -extern const unsigned MD5_CTX_sz; -extern const unsigned MD5_return_length; - -#define SHA2_EXTERN(LEN) \ - extern const unsigned SHA##LEN##_CTX_sz; \ - extern const unsigned SHA##LEN##_return_length; \ - extern const unsigned SHA##LEN##_block_length; \ - extern const unsigned SHA##LEN##_digest_length - -SHA2_EXTERN(224); -SHA2_EXTERN(256); -SHA2_EXTERN(384); -SHA2_EXTERN(512); - -#undef SHA2_EXTERN - struct __sanitizer_cap_rights { u64 cr_rights[2]; }; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp index aacd28c55cea..435f3b2861dc 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp @@ -498,7 +498,6 @@ struct urio_command { #include <md5.h> #include <rmd160.h> #include <soundcard.h> -#include <term.h> #include <termios.h> #include <time.h> #include <ttyent.h> @@ -515,7 +514,7 @@ struct urio_command { #include <stringlist.h> #if defined(__x86_64__) -#include <nvmm.h> +#include <dev/nvmm/nvmm_ioctl.h> #endif // clang-format on diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp index 7a89bf1c7498..ea8cc306268c 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -779,16 +779,16 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_SOUND_PCM_WRITE_FILTER = SOUND_PCM_WRITE_FILTER; #endif // SOUND_VERSION unsigned IOCTL_TCFLSH = TCFLSH; - unsigned IOCTL_TCGETA = TCGETA; +# if SANITIZER_TERMIOS_IOCTL_CONSTANTS unsigned IOCTL_TCGETS = TCGETS; +# endif unsigned IOCTL_TCSBRK = TCSBRK; unsigned IOCTL_TCSBRKP = TCSBRKP; - unsigned IOCTL_TCSETA = TCSETA; - unsigned IOCTL_TCSETAF = TCSETAF; - unsigned IOCTL_TCSETAW = TCSETAW; +# if SANITIZER_TERMIOS_IOCTL_CONSTANTS unsigned IOCTL_TCSETS = TCSETS; unsigned IOCTL_TCSETSF = TCSETSF; unsigned IOCTL_TCSETSW = TCSETSW; +# endif unsigned IOCTL_TCXONC = TCXONC; unsigned IOCTL_TIOCGLCKTRMIOS = TIOCGLCKTRMIOS; unsigned IOCTL_TIOCGSOFTCAR = TIOCGSOFTCAR; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index a2b6c37d5450..24966523f3a0 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -102,6 +102,8 @@ const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID ? FIRST_32_SECOND_64(104, 128) # if defined(_ABIN32) && _MIPS_SIM == _ABIN32 : FIRST_32_SECOND_64(176, 216); +# elif SANITIZER_MUSL + : FIRST_32_SECOND_64(160, 208); # else : FIRST_32_SECOND_64(160, 216); # endif @@ -476,6 +478,30 @@ struct __sanitizer_cmsghdr { int cmsg_level; int cmsg_type; }; +# elif SANITIZER_MUSL +struct __sanitizer_msghdr { + void *msg_name; + unsigned msg_namelen; + struct __sanitizer_iovec *msg_iov; + int msg_iovlen; +# if SANITIZER_WORDSIZE == 64 + int __pad1; +# endif + void *msg_control; + unsigned msg_controllen; +# if SANITIZER_WORDSIZE == 64 + int __pad2; +# endif + int msg_flags; +}; +struct __sanitizer_cmsghdr { + unsigned cmsg_len; +# if SANITIZER_WORDSIZE == 64 + int __pad1; +# endif + int cmsg_level; + int cmsg_type; +}; # else // In POSIX, int msg_iovlen; socklen_t msg_controllen; socklen_t cmsg_len; but // many implementations don't conform to the standard. @@ -1312,16 +1338,14 @@ extern unsigned IOCTL_SNDCTL_COPR_SENDMSG; extern unsigned IOCTL_SNDCTL_COPR_WCODE; extern unsigned IOCTL_SNDCTL_COPR_WDATA; extern unsigned IOCTL_TCFLSH; -extern unsigned IOCTL_TCGETA; -extern unsigned IOCTL_TCGETS; extern unsigned IOCTL_TCSBRK; extern unsigned IOCTL_TCSBRKP; -extern unsigned IOCTL_TCSETA; -extern unsigned IOCTL_TCSETAF; -extern unsigned IOCTL_TCSETAW; +# if SANITIZER_TERMIOS_IOCTL_CONSTANTS +extern unsigned IOCTL_TCGETS; extern unsigned IOCTL_TCSETS; extern unsigned IOCTL_TCSETSF; extern unsigned IOCTL_TCSETSW; +# endif extern unsigned IOCTL_TCXONC; extern unsigned IOCTL_TIOCGLCKTRMIOS; extern unsigned IOCTL_TIOCGSOFTCAR; diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S index 7d920bee4a2d..f1d11a3e7f54 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -222,6 +222,6 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) NO_EXEC_STACK_DIRECTIVE -GNU_PROPERTY_BTI_PAC +GNU_PROPERTY_BTI_PAC_GCS #endif diff --git a/libcxx/include/__algorithm/sort.h b/libcxx/include/__algorithm/sort.h index 06cb5b8ce705..8aa894e9228c 100644 --- a/libcxx/include/__algorithm/sort.h +++ b/libcxx/include/__algorithm/sort.h @@ -860,6 +860,9 @@ __sort<__less<long double>&, long double*>(long double*, long double*, __less<lo template <class _AlgPolicy, class _RandomAccessIterator, class _Comp> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __sort_dispatch(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp& __comp) { + if (__first == __last) // log(0) is undefined, so don't try computing the depth + return; + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; difference_type __depth_limit = 2 * std::__bit_log2(std::__to_unsigned_like(__last - __first)); diff --git a/libcxx/include/__assert b/libcxx/include/__assert index 90eaa6023587..a9451daf47f2 100644 --- a/libcxx/include/__assert +++ b/libcxx/include/__assert @@ -20,8 +20,8 @@ #define _LIBCPP_ASSERT(expression, message) \ (__builtin_expect(static_cast<bool>(expression), 1) \ ? (void)0 \ - : _LIBCPP_ASSERTION_HANDLER(__FILE__ ":" _LIBCPP_TOSTRING(__LINE__) ": assertion " _LIBCPP_TOSTRING( \ - expression) " failed: " message "\n")) + : _LIBCPP_ASSERTION_HANDLER(__FILE__ ":" _LIBCPP_TOSTRING( \ + __LINE__) ": libc++ Hardening assertion " _LIBCPP_TOSTRING(expression) " failed: " message "\n")) // WARNING: __builtin_assume can currently inhibit optimizations. Only add assumptions with a clear // optimization intent. See https://discourse.llvm.org/t/llvm-assume-blocks-optimization/71609 for a diff --git a/libcxx/include/__bit/bit_log2.h b/libcxx/include/__bit/bit_log2.h index 8077cd91d6fd..9ceeec1b2bc9 100644 --- a/libcxx/include/__bit/bit_log2.h +++ b/libcxx/include/__bit/bit_log2.h @@ -9,6 +9,7 @@ #ifndef _LIBCPP___BIT_BIT_LOG2_H #define _LIBCPP___BIT_BIT_LOG2_H +#include <__assert> #include <__bit/countl.h> #include <__config> #include <__type_traits/integer_traits.h> @@ -23,6 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD template <class _Tp> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __bit_log2(_Tp __t) _NOEXCEPT { static_assert(__is_unsigned_integer_v<_Tp>, "__bit_log2 requires an unsigned integer type"); + _LIBCPP_ASSERT_INTERNAL(__t != 0, "logarithm of 0 is undefined"); return numeric_limits<_Tp>::digits - 1 - std::__countl_zero(__t); } diff --git a/libcxx/include/__config b/libcxx/include/__config index d940461c3023..d6c6e6a08c08 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -28,7 +28,7 @@ // _LIBCPP_VERSION represents the version of libc++, which matches the version of LLVM. // Given a LLVM release LLVM XX.YY.ZZ (e.g. LLVM 17.0.1 == 17.00.01), _LIBCPP_VERSION is // defined to XXYYZZ. -# define _LIBCPP_VERSION 210000 +# define _LIBCPP_VERSION 210107 # define _LIBCPP_CONCAT_IMPL(_X, _Y) _X##_Y # define _LIBCPP_CONCAT(_X, _Y) _LIBCPP_CONCAT_IMPL(_X, _Y) @@ -38,11 +38,47 @@ # define _LIBCPP_FREESTANDING # endif +// NOLINTNEXTLINE(libcpp-cpp-version-check) +# if __cplusplus < 201103L +# define _LIBCPP_CXX03_LANG +# endif + +# if __has_feature(experimental_library) +# ifndef _LIBCPP_ENABLE_EXPERIMENTAL +# define _LIBCPP_ENABLE_EXPERIMENTAL +# endif +# endif + +// Incomplete features get their own specific disabling flags. This makes it +// easier to grep for target specific flags once the feature is complete. +# if defined(_LIBCPP_ENABLE_EXPERIMENTAL) || defined(_LIBCPP_BUILDING_LIBRARY) +# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 1 +# else +# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 0 +# endif + +# define _LIBCPP_HAS_EXPERIMENTAL_PSTL _LIBCPP_HAS_EXPERIMENTAL_LIBRARY +# define _LIBCPP_HAS_EXPERIMENTAL_TZDB _LIBCPP_HAS_EXPERIMENTAL_LIBRARY +# define _LIBCPP_HAS_EXPERIMENTAL_SYNCSTREAM _LIBCPP_HAS_EXPERIMENTAL_LIBRARY +# define _LIBCPP_HAS_EXPERIMENTAL_HARDENING_OBSERVE_SEMANTIC _LIBCPP_HAS_EXPERIMENTAL_LIBRARY + // HARDENING { -// TODO: Remove in LLVM 21. We're making this an error to catch folks who might not have migrated. -# ifdef _LIBCPP_ENABLE_ASSERTIONS -# error "_LIBCPP_ENABLE_ASSERTIONS has been removed, please use _LIBCPP_HARDENING_MODE instead" +// TODO(LLVM 23): Remove this. We're making these an error to catch folks who might not have migrated. +// Since hardening went through several changes (many of which impacted user-facing macros), +// we're keeping these checks around for a bit longer than usual. Failure to properly configure +// hardening results in checks being dropped silently, which is a pretty big deal. +# if defined(_LIBCPP_ENABLE_ASSERTIONS) +# error "_LIBCPP_ENABLE_ASSERTIONS has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)" +# endif +# if defined(_LIBCPP_ENABLE_HARDENED_MODE) +# error "_LIBCPP_ENABLE_HARDENED_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)" +# endif +# if defined(_LIBCPP_ENABLE_SAFE_MODE) +# error "_LIBCPP_ENABLE_SAFE_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)" +# endif +# if defined(_LIBCPP_ENABLE_DEBUG_MODE) +# error "_LIBCPP_ENABLE_DEBUG_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)" # endif // The library provides the macro `_LIBCPP_HARDENING_MODE` which can be set to one of the following values: @@ -147,16 +183,53 @@ _LIBCPP_HARDENING_MODE_EXTENSIVE, \ _LIBCPP_HARDENING_MODE_DEBUG # endif +// Hardening assertion semantics generally mirror the evaluation semantics of C++26 Contracts: +// - `ignore` evaluates the assertion but doesn't do anything if it fails (note that it differs from the Contracts +// `ignore` semantic which wouldn't evaluate the assertion at all); +// - `observe` logs an error (indicating, if possible, that the error is fatal) and continues execution; +// - `quick-enforce` terminates the program as fast as possible (via trapping); +// - `enforce` logs an error and then terminates the program. +// +// Notes: +// - Continuing execution after a hardening check fails results in undefined behavior; the `observe` semantic is meant +// to make adopting hardening easier but should not be used outside of this scenario; +// - C++26 wording for Library Hardening precludes a conforming Hardened implementation from using the Contracts +// `ignore` semantic when evaluating hardened preconditions in the Library. Libc++ allows using this semantic for +// hardened preconditions, however, be aware that using `ignore` does not produce a conforming "Hardened" +// implementation, unlike the other semantics above. +// clang-format off +# define _LIBCPP_ASSERTION_SEMANTIC_IGNORE (1 << 1) +# define _LIBCPP_ASSERTION_SEMANTIC_OBSERVE (1 << 2) +# define _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE (1 << 3) +# define _LIBCPP_ASSERTION_SEMANTIC_ENFORCE (1 << 4) +// clang-format on + +// Allow users to define an arbitrary assertion semantic; otherwise, use the default mapping from modes to semantics. +// The default is for production-capable modes to use `quick-enforce` (i.e., trap) and for the `debug` mode to use +// `enforce` (i.e., log and abort). +# ifndef _LIBCPP_ASSERTION_SEMANTIC + +# if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG +# define _LIBCPP_ASSERTION_SEMANTIC _LIBCPP_ASSERTION_SEMANTIC_ENFORCE +# else +# define _LIBCPP_ASSERTION_SEMANTIC _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE +# endif + +# else +# if !_LIBCPP_HAS_EXPERIMENTAL_LIBRARY +# error "Assertion semantics are an experimental feature." +# endif +# if defined(_LIBCPP_CXX03_LANG) +# error "Assertion semantics are not available in the C++03 mode." +# endif + +# endif // _LIBCPP_ASSERTION_SEMANTIC + // } HARDENING # define _LIBCPP_TOSTRING2(x) #x # define _LIBCPP_TOSTRING(x) _LIBCPP_TOSTRING2(x) -// NOLINTNEXTLINE(libcpp-cpp-version-check) -# if __cplusplus < 201103L -# define _LIBCPP_CXX03_LANG -# endif - # ifndef __has_constexpr_builtin # define __has_constexpr_builtin(x) 0 # endif @@ -190,24 +263,6 @@ _LIBCPP_HARDENING_MODE_DEBUG # define _LIBCPP_ABI_VCRUNTIME # endif -# if __has_feature(experimental_library) -# ifndef _LIBCPP_ENABLE_EXPERIMENTAL -# define _LIBCPP_ENABLE_EXPERIMENTAL -# endif -# endif - -// Incomplete features get their own specific disabling flags. This makes it -// easier to grep for target specific flags once the feature is complete. -# if defined(_LIBCPP_ENABLE_EXPERIMENTAL) || defined(_LIBCPP_BUILDING_LIBRARY) -# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 1 -# else -# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 0 -# endif - -# define _LIBCPP_HAS_EXPERIMENTAL_PSTL _LIBCPP_HAS_EXPERIMENTAL_LIBRARY -# define _LIBCPP_HAS_EXPERIMENTAL_TZDB _LIBCPP_HAS_EXPERIMENTAL_LIBRARY -# define _LIBCPP_HAS_EXPERIMENTAL_SYNCSTREAM _LIBCPP_HAS_EXPERIMENTAL_LIBRARY - # if defined(__MVS__) # include <features.h> // for __NATIVE_ASCII_F # endif diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index a75cd0a67533..3f758d97394f 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -30,8 +30,20 @@ #elif _LIBCPP_ABI_FORCE_MICROSOFT # define _LIBCPP_ABI_MICROSOFT #else +// Windows uses the Microsoft ABI # if defined(_WIN32) && defined(_MSC_VER) # define _LIBCPP_ABI_MICROSOFT + +// 32-bit ARM uses the Itanium ABI with a few differences (array cookies, etc), +// and so does 64-bit ARM on Apple platforms. +# elif defined(__arm__) || (defined(__APPLE__) && defined(__aarch64__)) +# define _LIBCPP_ABI_ITANIUM_WITH_ARM_DIFFERENCES + +// Non-Apple 64-bit ARM uses the vanilla Itanium ABI +# elif defined(__aarch64__) +# define _LIBCPP_ABI_ITANIUM + +// We assume that other architectures also use the vanilla Itanium ABI too # else # define _LIBCPP_ABI_ITANIUM # endif diff --git a/libcxx/include/__cxx03/__config b/libcxx/include/__cxx03/__config index ef47327d9635..9b88a495055e 100644 --- a/libcxx/include/__cxx03/__config +++ b/libcxx/include/__cxx03/__config @@ -152,6 +152,10 @@ _LIBCPP_HARDENING_MODE_EXTENSIVE, \ _LIBCPP_HARDENING_MODE_DEBUG # endif +# ifdef _LIBCPP_ASSERTION_SEMANTIC +# error "Assertion semantics are not available in the C++03 mode." +# endif + // } HARDENING # define _LIBCPP_TOSTRING2(x) #x diff --git a/libcxx/include/__functional/hash.h b/libcxx/include/__functional/hash.h index 489a6f00b8a3..83bbf1b5e26c 100644 --- a/libcxx/include/__functional/hash.h +++ b/libcxx/include/__functional/hash.h @@ -21,6 +21,7 @@ #include <__type_traits/is_enum.h> #include <__type_traits/is_floating_point.h> #include <__type_traits/is_integral.h> +#include <__type_traits/is_unqualified.h> #include <__type_traits/underlying_type.h> #include <__utility/pair.h> #include <__utility/swap.h> @@ -355,7 +356,8 @@ struct __hash_impl { }; template <class _Tp> -struct __hash_impl<_Tp, __enable_if_t<is_enum<_Tp>::value> > : __unary_function<_Tp, size_t> { +struct __hash_impl<_Tp, __enable_if_t<is_enum<_Tp>::value && __is_unqualified_v<_Tp> > > + : __unary_function<_Tp, size_t> { _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { using type = __underlying_type_t<_Tp>; return hash<type>()(static_cast<type>(__v)); @@ -363,17 +365,21 @@ struct __hash_impl<_Tp, __enable_if_t<is_enum<_Tp>::value> > : __unary_function< }; template <class _Tp> -struct __hash_impl<_Tp, __enable_if_t<is_integral<_Tp>::value && (sizeof(_Tp) <= sizeof(size_t))> > +struct __hash_impl< + _Tp, + __enable_if_t<is_integral<_Tp>::value && __is_unqualified_v<_Tp> && (sizeof(_Tp) <= sizeof(size_t))> > : __unary_function<_Tp, size_t> { _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { return static_cast<size_t>(__v); } }; template <class _Tp> -struct __hash_impl<_Tp, __enable_if_t<is_integral<_Tp>::value && (sizeof(_Tp) > sizeof(size_t))> > +struct __hash_impl<_Tp, + __enable_if_t<is_integral<_Tp>::value && __is_unqualified_v<_Tp> && (sizeof(_Tp) > sizeof(size_t))> > : __scalar_hash<_Tp> {}; template <class _Tp> -struct __hash_impl<_Tp, __enable_if_t<is_floating_point<_Tp>::value> > : __scalar_hash<_Tp> { +struct __hash_impl<_Tp, __enable_if_t<is_floating_point<_Tp>::value && __is_unqualified_v<_Tp> > > + : __scalar_hash<_Tp> { _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { // -0.0 and 0.0 should return same hash if (__v == 0.0f) diff --git a/libcxx/include/__log_hardening_failure b/libcxx/include/__log_hardening_failure new file mode 100644 index 000000000000..d1805306f6b6 --- /dev/null +++ b/libcxx/include/__log_hardening_failure @@ -0,0 +1,42 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___LOG_HARDENING_FAILURE +#define _LIBCPP___LOG_HARDENING_FAILURE + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +// Hardening logging is not available in the C++03 mode; moreover, it is currently only available in the experimental +// library. +#if _LIBCPP_HAS_EXPERIMENTAL_HARDENING_OBSERVE_SEMANTIC && !defined(_LIBCPP_CXX03_LANG) + +_LIBCPP_BEGIN_NAMESPACE_STD + +// This function should never be called directly from the code -- it should only be called through the +// `_LIBCPP_LOG_HARDENING_FAILURE` macro. +[[__gnu__::__cold__]] _LIBCPP_EXPORTED_FROM_ABI void __log_hardening_failure(const char* __message) noexcept; + +// _LIBCPP_LOG_HARDENING_FAILURE(message) +// +// This macro is used to log an error without terminating the program (as is the case for hardening failures if the +// `observe` assertion semantic is used). + +# if !defined(_LIBCPP_LOG_HARDENING_FAILURE) +# define _LIBCPP_LOG_HARDENING_FAILURE(__message) ::std::__log_hardening_failure(__message) +# endif // !defined(_LIBCPP_LOG_HARDENING_FAILURE) + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_HAS_EXPERIMENTAL_HARDENING_OBSERVE_SEMANTIC && !defined(_LIBCPP_CXX03_LANG) + +#endif // _LIBCPP___LOG_HARDENING_FAILURE diff --git a/libcxx/include/__memory/array_cookie.h b/libcxx/include/__memory/array_cookie.h index 806a9e99ecaf..be59f365aa80 100644 --- a/libcxx/include/__memory/array_cookie.h +++ b/libcxx/include/__memory/array_cookie.h @@ -13,6 +13,7 @@ #include <__config> #include <__configuration/abi.h> #include <__cstddef/size_t.h> +#include <__memory/addressof.h> #include <__type_traits/integral_constant.h> #include <__type_traits/is_trivially_destructible.h> #include <__type_traits/negation.h> @@ -26,14 +27,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD // Trait representing whether a type requires an array cookie at the start of its allocation when // allocated as `new T[n]` and deallocated as `delete[] array`. // -// Under the Itanium C++ ABI [1], we know that an array cookie is available unless `T` is trivially -// destructible and the call to `operator delete[]` is not a sized operator delete. Under ABIs other -// than the Itanium ABI, we assume there are no array cookies. +// Under the Itanium C++ ABI [1] and the ARM ABI which derives from it, we know that an array cookie is available +// unless `T` is trivially destructible and the call to `operator delete[]` is not a sized operator delete. Under +// other ABIs, we assume there are no array cookies. // // [1]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#array-cookies -#ifdef _LIBCPP_ABI_ITANIUM +#if defined(_LIBCPP_ABI_ITANIUM) || defined(_LIBCPP_ABI_ITANIUM_WITH_ARM_DIFFERENCES) // TODO: Use a builtin instead -// TODO: We should factor in the choice of the usual deallocation function in this determination. +// TODO: We should factor in the choice of the usual deallocation function in this determination: +// a cookie may be available in more cases but we ignore those for now. template <class _Tp> struct __has_array_cookie : _Not<is_trivially_destructible<_Tp> > {}; #else @@ -41,13 +43,79 @@ template <class _Tp> struct __has_array_cookie : false_type {}; #endif +struct __itanium_array_cookie { + size_t __element_count; +}; + +template <class _Tp> +struct [[__gnu__::__aligned__(_LIBCPP_ALIGNOF(_Tp))]] __arm_array_cookie { + size_t __element_size; + size_t __element_count; +}; + +// Return the element count in the array cookie located before the given pointer. +// +// In the Itanium ABI [1] +// ---------------------- +// The element count is stored immediately before the first element of the array. If the preferred alignment +// of array elements (which is different from the ABI alignment) is more than that of size_t, additional +// padding bytes exist before the array cookie. Assuming array elements of size and alignment 16 bytes, that +// gives us the following layout: +// +// |ooooooooxxxxxxxxaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbccccccccccccccccdddddddddddddddd| +// ^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// | ^^^^^^^^ | +// | | array elements +// padding | +// element count +// +// +// In the Itanium ABI with ARM differences [2] +// ------------------------------------------- +// The array cookie is stored at the very start of the allocation and it has the following form: +// +// struct array_cookie { +// std::size_t element_size; // element_size != 0 +// std::size_t element_count; +// }; +// +// Assuming elements of size and alignment 32 bytes, this gives us the following layout: +// +// |xxxxxxxxXXXXXXXXooooooooooooooooaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb| +// ^^^^^^^^ ^^^^^^^^^^^^^^^^ +// | ^^^^^^^^ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// element size | padding | +// element count array elements +// +// We must be careful to take into account the alignment of the array cookie, which may result in padding +// bytes between the element count and the first element of the array. Note that for ARM, the compiler +// aligns the array cookie using the ABI alignment, not the preferred alignment of array elements. +// +// [1]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#array-cookies +// [2]: https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Handle-C++-differences template <class _Tp> // Avoid failures when -fsanitize-address-poison-custom-array-cookie is enabled -_LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") size_t __get_array_cookie(_Tp const* __ptr) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") size_t __get_array_cookie([[__maybe_unused__]] _Tp const* __ptr) { static_assert( __has_array_cookie<_Tp>::value, "Trying to access the array cookie of a type that is not guaranteed to have one"); - size_t const* __cookie = reinterpret_cast<size_t const*>(__ptr) - 1; // TODO: Use a builtin instead - return *__cookie; + +#if defined(_LIBCPP_ABI_ITANIUM) + using _ArrayCookie = __itanium_array_cookie; +#elif defined(_LIBCPP_ABI_ITANIUM_WITH_ARM_DIFFERENCES) + using _ArrayCookie = __arm_array_cookie<_Tp>; +#else + static_assert(false, "The array cookie layout is unknown on this ABI"); + struct _ArrayCookie { // dummy definition required to make the function parse + size_t element_count; + }; +#endif + + char const* __array_cookie_start = reinterpret_cast<char const*>(__ptr) - sizeof(_ArrayCookie); + _ArrayCookie __cookie; + // This is necessary to avoid violating strict aliasing. It's valid because _ArrayCookie is an + // implicit lifetime type. + __builtin_memcpy(std::addressof(__cookie), __array_cookie_start, sizeof(_ArrayCookie)); + return __cookie.__element_count; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__memory/compressed_pair.h b/libcxx/include/__memory/compressed_pair.h index fb7b7b7afcc8..29e503931b0b 100644 --- a/libcxx/include/__memory/compressed_pair.h +++ b/libcxx/include/__memory/compressed_pair.h @@ -80,21 +80,45 @@ class __compressed_pair_padding { template <class _ToPad> class __compressed_pair_padding<_ToPad, true> {}; -# define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2) \ - _LIBCPP_NO_UNIQUE_ADDRESS __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1; \ - _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \ - _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \ - _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _) - -# define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3) \ - _LIBCPP_NO_UNIQUE_ADDRESS \ - __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>), \ - __aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1; \ - _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \ - _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \ - _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _); \ - _LIBCPP_NO_UNIQUE_ADDRESS T3 Initializer3; \ - _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T3> _LIBCPP_CONCAT3(__padding3_, __LINE__, _) +// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed +# ifdef _LIBCPP_COMPILER_GCC +# define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2) \ + _LIBCPP_NO_UNIQUE_ADDRESS __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \ + _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _) + +# define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3) \ + _LIBCPP_NO_UNIQUE_ADDRESS \ + __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>), \ + __aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \ + _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _); \ + _LIBCPP_NO_UNIQUE_ADDRESS T3 Initializer3; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T3> _LIBCPP_CONCAT3(__padding3_, __LINE__, _) +# else +# define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2) \ + struct { \ + _LIBCPP_NO_UNIQUE_ADDRESS \ + __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \ + _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _); \ + } + +# define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3) \ + struct { \ + _LIBCPP_NO_UNIQUE_ADDRESS \ + __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>), \ + __aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \ + _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _); \ + _LIBCPP_NO_UNIQUE_ADDRESS T3 Initializer3; \ + _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T3> _LIBCPP_CONCAT3(__padding3_, __LINE__, _); \ + } +# endif #else # define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2) \ diff --git a/libcxx/include/__ranges/join_view.h b/libcxx/include/__ranges/join_view.h index 327b349f476a..364f056d8d2c 100644 --- a/libcxx/include/__ranges/join_view.h +++ b/libcxx/include/__ranges/join_view.h @@ -410,8 +410,13 @@ struct __segmented_iterator_traits<_JoinViewIterator> { static constexpr _LIBCPP_HIDE_FROM_ABI _JoinViewIterator __compose(__segment_iterator __seg_iter, __local_iterator __local_iter) { - return _JoinViewIterator( - std::move(__seg_iter).__get_data(), std::move(__seg_iter).__get_iter(), std::move(__local_iter)); + auto&& __parent = std::move(__seg_iter).__get_data(); + auto&& __outer = std::move(__seg_iter).__get_iter(); + if (__local_iter == ranges::end(*__outer)) { + ++__outer; + return _JoinViewIterator(*__parent, __outer); + } + return _JoinViewIterator(__parent, __outer, std::move(__local_iter)); } }; diff --git a/libcxx/include/__type_traits/invoke.h b/libcxx/include/__type_traits/invoke.h index 5ff2efbe5faa..3f5626c01443 100644 --- a/libcxx/include/__type_traits/invoke.h +++ b/libcxx/include/__type_traits/invoke.h @@ -67,21 +67,21 @@ _LIBCPP_BEGIN_NAMESPACE_STD #if __has_builtin(__builtin_invoke) -template <class... _Args> -using __invoke_result_t _LIBCPP_NODEBUG = decltype(__builtin_invoke(std::declval<_Args>()...)); - template <class, class... _Args> struct __invoke_result_impl {}; template <class... _Args> -struct __invoke_result_impl<__void_t<__invoke_result_t<_Args...> >, _Args...> { - using type _LIBCPP_NODEBUG = __invoke_result_t<_Args...>; +struct __invoke_result_impl<__void_t<decltype(__builtin_invoke(std::declval<_Args>()...))>, _Args...> { + using type _LIBCPP_NODEBUG = decltype(__builtin_invoke(std::declval<_Args>()...)); }; template <class... _Args> using __invoke_result _LIBCPP_NODEBUG = __invoke_result_impl<void, _Args...>; template <class... _Args> +using __invoke_result_t _LIBCPP_NODEBUG = typename __invoke_result<_Args...>::type; + +template <class... _Args> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __invoke_result_t<_Args...> __invoke(_Args&&... __args) _NOEXCEPT_(noexcept(__builtin_invoke(std::forward<_Args>(__args)...))) { return __builtin_invoke(std::forward<_Args>(__args)...); diff --git a/libcxx/include/__type_traits/is_unqualified.h b/libcxx/include/__type_traits/is_unqualified.h new file mode 100644 index 000000000000..7970b3611601 --- /dev/null +++ b/libcxx/include/__type_traits/is_unqualified.h @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___TYPE_TRAITS_IS_UNQUALIFIED_H +#define _LIBCPP___TYPE_TRAITS_IS_UNQUALIFIED_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp> +inline const bool __is_unqualified_v = __is_same(_Tp, __remove_cvref(_Tp)); + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___TYPE_TRAITS_IS_UNQUALIFIED_H diff --git a/libcxx/include/ext/hash_map b/libcxx/include/ext/hash_map index d6b92204f437..46815eaffa8b 100644 --- a/libcxx/include/ext/hash_map +++ b/libcxx/include/ext/hash_map @@ -744,7 +744,7 @@ public: _LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return __table_.begin(); } _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __table_.end(); } - _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__emplace_unique(__x); } + _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__emplace_multi(__x); } _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x); } template <class _InputIterator> _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last); @@ -831,7 +831,7 @@ template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc> template <class _InputIterator> inline void hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) { for (; __first != __last; ++__first) - __table_.__emplace_unique(*__first); + __table_.__emplace_multi(*__first); } template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc> diff --git a/libcxx/include/ext/hash_set b/libcxx/include/ext/hash_set index 7fd5df24ed3a..62a7a0dbcffb 100644 --- a/libcxx/include/ext/hash_set +++ b/libcxx/include/ext/hash_set @@ -458,7 +458,7 @@ public: _LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return __table_.begin(); } _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __table_.end(); } - _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__emplace_unique(__x); } + _LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__emplace_multi(__x); } _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator, const value_type& __x) { return insert(__x); } template <class _InputIterator> _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last); @@ -543,7 +543,7 @@ template <class _Value, class _Hash, class _Pred, class _Alloc> template <class _InputIterator> inline void hash_multiset<_Value, _Hash, _Pred, _Alloc>::insert(_InputIterator __first, _InputIterator __last) { for (; __first != __last; ++__first) - __table_.__emplace_unique(*__first); + __table_.__emplace_multi(*__first); } template <class _Value, class _Hash, class _Pred, class _Alloc> diff --git a/libcxx/include/fstream b/libcxx/include/fstream index c86f709bedb8..6d3f20fff688 100644 --- a/libcxx/include/fstream +++ b/libcxx/include/fstream @@ -401,6 +401,14 @@ private: } } } + + _LIBCPP_HIDE_FROM_ABI typename traits_type::int_type __overflow_failed() { + if (this->pptr() == this->epptr() + 1) { + this->pbump(-1); // lose the character we overflowed above -- we don't really have a + // choice since we couldn't commit the contents of the put area + } + return traits_type::eof(); + } }; template <class _CharT, class _Traits> @@ -841,8 +849,9 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits> if (__always_noconv_) { size_t __n = static_cast<size_t>(this->pptr() - this->pbase()); - if (std::fwrite(this->pbase(), sizeof(char_type), __n, __file_) != __n) - return traits_type::eof(); + if (std::fwrite(this->pbase(), sizeof(char_type), __n, __file_) != __n) { + return __overflow_failed(); + } } else { if (!__cv_) std::__throw_bad_cast(); @@ -854,34 +863,38 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits> char* __extbuf_end = __extbuf_; do { codecvt_base::result __r = __cv_->out(__st_, __b, __p, __end, __extbuf_, __extbuf_ + __ebs_, __extbuf_end); - if (__end == __b) - return traits_type::eof(); + if (__end == __b) { + return __overflow_failed(); + } // No conversion needed: output characters directly to the file, done. if (__r == codecvt_base::noconv) { size_t __n = static_cast<size_t>(__p - __b); - if (std::fwrite(__b, 1, __n, __file_) != __n) - return traits_type::eof(); + if (std::fwrite(__b, 1, __n, __file_) != __n) { + return __overflow_failed(); + } break; // Conversion successful: output the converted characters to the file, done. } else if (__r == codecvt_base::ok) { size_t __n = static_cast<size_t>(__extbuf_end - __extbuf_); - if (std::fwrite(__extbuf_, 1, __n, __file_) != __n) - return traits_type::eof(); + if (std::fwrite(__extbuf_, 1, __n, __file_) != __n) { + return __overflow_failed(); + } break; // Conversion partially successful: output converted characters to the file and repeat with the // remaining characters. } else if (__r == codecvt_base::partial) { size_t __n = static_cast<size_t>(__extbuf_end - __extbuf_); - if (std::fwrite(__extbuf_, 1, __n, __file_) != __n) - return traits_type::eof(); + if (std::fwrite(__extbuf_, 1, __n, __file_) != __n) { + return __overflow_failed(); + } __b = const_cast<char_type*>(__end); continue; } else { - return traits_type::eof(); + return __overflow_failed(); } } while (true); } diff --git a/libcxx/include/map b/libcxx/include/map index 225156580147..3d88b32dd426 100644 --- a/libcxx/include/map +++ b/libcxx/include/map @@ -692,12 +692,12 @@ public: # if _LIBCPP_STD_VER >= 14 template <typename _K2> _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const { - return __comp_(__x, __y.__get_value().first); + return __comp_(__x, __y.first); } template <typename _K2> _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const { - return __comp_(__x.__get_value().first, __y); + return __comp_(__x.first, __y); } # endif }; diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in index 61ba1c381b2b..4509dc20c6e6 100644 --- a/libcxx/include/module.modulemap.in +++ b/libcxx/include/module.modulemap.in @@ -336,6 +336,7 @@ module std_core [system] { header "__type_traits/is_union.h" export std_core.type_traits.integral_constant } + module is_unqualified { header "__type_traits/is_unqualified.h" } module is_unsigned { header "__type_traits/is_unsigned.h" export std_core.type_traits.integral_constant @@ -2353,6 +2354,9 @@ module std [system] { header "__std_mbstate_t.h" export * } + module log_hardening_failure { + header "__log_hardening_failure" + } module verbose_abort { header "__verbose_abort" } diff --git a/libcxx/include/string b/libcxx/include/string index 514dd91c7c17..d282071abf11 100644 --- a/libcxx/include/string +++ b/libcxx/include/string @@ -974,7 +974,12 @@ public: _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string() _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value) - : __rep_() { +# if _LIBCPP_STD_VER >= 20 // TODO(LLVM 23): Remove this condition; this is a workaround for https://llvm.org/PR154567 + : __rep_(__short()) +# else + : __rep_() +# endif + { __annotate_new(0); } @@ -984,7 +989,12 @@ public: # else _NOEXCEPT # endif - : __rep_(), __alloc_(__a) { +# if _LIBCPP_STD_VER >= 20 // TODO(LLVM 23): Remove this condition; this is a workaround for https://llvm.org/PR154567 + : __rep_(__short()), +# else + : __rep_(), +# endif + __alloc_(__a) { __annotate_new(0); } diff --git a/libcxx/src/algorithm.cpp b/libcxx/src/algorithm.cpp index d388fee5f99c..8157be6f7406 100644 --- a/libcxx/src/algorithm.cpp +++ b/libcxx/src/algorithm.cpp @@ -13,6 +13,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template <class Comp, class RandomAccessIterator> void __sort(RandomAccessIterator first, RandomAccessIterator last, Comp comp) { + if (first == last) // log(0) is undefined, so don't try computing the depth + return; + auto depth_limit = 2 * std::__bit_log2(static_cast<size_t>(last - first)); // Only use bitset partitioning for arithmetic types. We should also check diff --git a/libcxx/src/experimental/log_hardening_failure.cpp b/libcxx/src/experimental/log_hardening_failure.cpp new file mode 100644 index 000000000000..f836c1545224 --- /dev/null +++ b/libcxx/src/experimental/log_hardening_failure.cpp @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__config> +#include <__log_hardening_failure> +#include <cstdio> + +#ifdef __BIONIC__ +# include <syslog.h> +#endif // __BIONIC__ + +_LIBCPP_BEGIN_NAMESPACE_STD + +void __log_hardening_failure(const char* message) noexcept { + // Always log the message to `stderr` in case the platform-specific system calls fail. + std::fputs(message, stderr); + +#if defined(__BIONIC__) + // Show error in logcat. The latter two arguments are ignored on Android. + openlog("libc++", 0, 0); + syslog(LOG_CRIT, "%s", message); + closelog(); +#endif +} + +_LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/vendor/llvm/default_assertion_handler.in b/libcxx/vendor/llvm/default_assertion_handler.in index f115658f9f3c..d352405e905b 100644 --- a/libcxx/vendor/llvm/default_assertion_handler.in +++ b/libcxx/vendor/llvm/default_assertion_handler.in @@ -16,6 +16,7 @@ # include <__cxx03/__verbose_trap> #else # include <__config> +# include <__log_hardening_failure> # include <__verbose_abort> # include <__verbose_trap> #endif @@ -24,14 +25,40 @@ # pragma GCC system_header #endif -#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG +#if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS) -# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message) +// Keep the old implementation that doesn't support assertion semantics for backward compatibility with the frozen C++03 +// mode. +# if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG +# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message) +# else +# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_TRAP(message) +# endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG #else -# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_TRAP(message) +# if _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_IGNORE +# define _LIBCPP_ASSERTION_HANDLER(message) ((void)0) + +# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_OBSERVE +# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_LOG_HARDENING_FAILURE(message) + +# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE +# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_TRAP(message) + +# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_ENFORCE +# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message) + +# else + +# error _LIBCPP_ASSERTION_SEMANTIC must be set to one of the following values: \ +_LIBCPP_ASSERTION_SEMANTIC_IGNORE, \ +_LIBCPP_ASSERTION_SEMANTIC_OBSERVE, \ +_LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE, \ +_LIBCPP_ASSERTION_SEMANTIC_ENFORCE + +# endif // _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_IGNORE -#endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG +#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS) #endif // _LIBCPP___ASSERTION_HANDLER diff --git a/libunwind/src/Unwind-seh.cpp b/libunwind/src/Unwind-seh.cpp index 8b83f10615f2..110c5987c3f1 100644 --- a/libunwind/src/Unwind-seh.cpp +++ b/libunwind/src/Unwind-seh.cpp @@ -174,7 +174,8 @@ _GCC_specific_handler(PEXCEPTION_RECORD ms_exc, PVOID frame, PCONTEXT ms_ctx, } // FIXME: Indicate target frame in foreign case! // phase 2: the clean up phase - RtlUnwindEx(frame, (PVOID)disp->ControlPc, ms_exc, exc, ms_ctx, disp->HistoryTable); + RtlUnwindEx(frame, (PVOID)disp->ControlPc, ms_exc, exc, disp->ContextRecord, + disp->HistoryTable); _LIBUNWIND_ABORT("RtlUnwindEx() failed"); case _URC_INSTALL_CONTEXT: { // If we were called by __libunwind_seh_personality(), indicate that diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp index 55db035e6204..9a1afd3721f5 100644 --- a/libunwind/src/UnwindCursor.hpp +++ b/libunwind/src/UnwindCursor.hpp @@ -173,7 +173,8 @@ bool DwarfFDECache<A>::_registeredForDyldUnloads = false; #endif template <typename A> -typename A::pint_t DwarfFDECache<A>::findFDE(pint_t mh, pint_t pc) { +typename DwarfFDECache<A>::pint_t DwarfFDECache<A>::findFDE(pint_t mh, + pint_t pc) { pint_t result = 0; _LIBUNWIND_LOG_IF_FALSE(_lock.lock_shared()); for (entry *p = _buffer; p < _bufferUsed; ++p) { diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S index 5e199188945d..1bcd205be260 100644 --- a/libunwind/src/UnwindRegistersRestore.S +++ b/libunwind/src/UnwindRegistersRestore.S @@ -1044,9 +1044,10 @@ DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind18Registers_mips_o326jumptoEv) lw $27, (4 * 27)($4) lw $28, (4 * 28)($4) lw $29, (4 * 29)($4) - lw $30, (4 * 30)($4) // load new pc into ra lw $31, (4 * 32)($4) + // MIPS 1 has load delay slot. Ensure lw $31 and jr are separated by an instruction. + lw $30, (4 * 30)($4) // jump to ra, load a0 in the delay slot jr $31 lw $4, (4 * 4)($4) @@ -1082,11 +1083,13 @@ DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind21Registers_mips_newabi6jumptoEv) ld $2, (8 * 2)($4) ld $3, (8 * 3)($4) // skip a0 for now - .irp i,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 + .irp i,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29 ld $\i, (8 * \i)($4) .endr // load new pc into ra ld $31, (8 * 32)($4) + // MIPS 1 has load delay slot. Ensure lw $31 and jr are separated by an instruction. + ld $30, (8 * 30)($4) // jump to ra, load a0 in the delay slot jr $31 ld $4, (8 * 4)($4) diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp index 01752cdc6a9d..cfb33daa024a 100644 --- a/lld/COFF/Chunks.cpp +++ b/lld/COFF/Chunks.cpp @@ -939,7 +939,7 @@ void ECCodeMapChunk::writeTo(uint8_t *buf) const { auto table = reinterpret_cast<chpe_range_entry *>(buf); for (uint32_t i = 0; i < map.size(); i++) { const ECCodeMapEntry &entry = map[i]; - uint32_t start = entry.first->getRVA(); + uint32_t start = entry.first->getRVA() & ~0xfff; table[i].StartOffset = start | entry.type; table[i].Length = entry.last->getRVA() + entry.last->getSize() - start; } diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h index 79b63e5b7236..91b6e632fa7e 100644 --- a/lld/COFF/Config.h +++ b/lld/COFF/Config.h @@ -192,6 +192,18 @@ struct Configuration { // Used for /lldltocachepolicy=policy llvm::CachePruningPolicy ltoCachePolicy; + // Used for /thinlto-distributor:<path> + StringRef dtltoDistributor; + + // Used for /thinlto-distributor-arg:<arg> + llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs; + + // Used for /thinlto-remote-compiler:<path> + StringRef dtltoCompiler; + + // Used for /thinlto-remote-compiler-arg:<arg> + llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs; + // Used for /opt:[no]ltodebugpassmanager bool ltoDebugPassManager = false; @@ -307,7 +319,7 @@ struct Configuration { bool warnDebugInfoUnusable = true; bool warnLongSectionNames = true; bool warnStdcallFixup = true; - bool warnExportedDllMain = true; + bool warnImportedDllMain = true; bool incremental = true; bool integrityCheck = false; bool killAt = false; diff --git a/lld/COFF/DLL.cpp b/lld/COFF/DLL.cpp index c327da28ce13..10bc898244a4 100644 --- a/lld/COFF/DLL.cpp +++ b/lld/COFF/DLL.cpp @@ -244,40 +244,36 @@ static const uint8_t thunkX64[] = { }; static const uint8_t tailMergeX64[] = { - 0x51, // push rcx - 0x52, // push rdx - 0x41, 0x50, // push r8 - 0x41, 0x51, // push r9 - 0x48, 0x83, 0xEC, 0x48, // sub rsp, 48h - 0x66, 0x0F, 0x7F, 0x04, 0x24, // movdqa xmmword ptr [rsp], xmm0 - 0x66, 0x0F, 0x7F, 0x4C, 0x24, 0x10, // movdqa xmmword ptr [rsp+10h], xmm1 - 0x66, 0x0F, 0x7F, 0x54, 0x24, 0x20, // movdqa xmmword ptr [rsp+20h], xmm2 - 0x66, 0x0F, 0x7F, 0x5C, 0x24, 0x30, // movdqa xmmword ptr [rsp+30h], xmm3 - 0x48, 0x8B, 0xD0, // mov rdx, rax - 0x48, 0x8D, 0x0D, 0, 0, 0, 0, // lea rcx, [___DELAY_IMPORT_...] - 0xE8, 0, 0, 0, 0, // call __delayLoadHelper2 - 0x66, 0x0F, 0x6F, 0x04, 0x24, // movdqa xmm0, xmmword ptr [rsp] - 0x66, 0x0F, 0x6F, 0x4C, 0x24, 0x10, // movdqa xmm1, xmmword ptr [rsp+10h] - 0x66, 0x0F, 0x6F, 0x54, 0x24, 0x20, // movdqa xmm2, xmmword ptr [rsp+20h] - 0x66, 0x0F, 0x6F, 0x5C, 0x24, 0x30, // movdqa xmm3, xmmword ptr [rsp+30h] - 0x48, 0x83, 0xC4, 0x48, // add rsp, 48h - 0x41, 0x59, // pop r9 - 0x41, 0x58, // pop r8 - 0x5A, // pop rdx - 0x59, // pop rcx - 0xFF, 0xE0, // jmp rax + 0x48, 0x89, 0x4C, 0x24, 0x08, // mov qword ptr [rsp+8], rcx + 0x48, 0x89, 0x54, 0x24, 0x10, // mov qword ptr [rsp+10h], rdx + 0x4C, 0x89, 0x44, 0x24, 0x18, // mov qword ptr [rsp+18h], r8 + 0x4C, 0x89, 0x4C, 0x24, 0x20, // mov qword ptr [rsp+20h], r9 + 0x48, 0x83, 0xEC, 0x68, // sub rsp, 68h + 0x66, 0x0F, 0x7F, 0x44, 0x24, 0x20, // movdqa xmmword ptr [rsp+20h], xmm0 + 0x66, 0x0F, 0x7F, 0x4C, 0x24, 0x30, // movdqa xmmword ptr [rsp+30h], xmm1 + 0x66, 0x0F, 0x7F, 0x54, 0x24, 0x40, // movdqa xmmword ptr [rsp+40h], xmm2 + 0x66, 0x0F, 0x7F, 0x5C, 0x24, 0x50, // movdqa xmmword ptr [rsp+50h], xmm3 + 0x48, 0x8B, 0xD0, // mov rdx, rax + 0x48, 0x8D, 0x0D, 0, 0, 0, 0, // lea rcx, [___DELAY_IMPORT_...] + 0xE8, 0, 0, 0, 0, // call __delayLoadHelper2 + 0x66, 0x0F, 0x6F, 0x44, 0x24, 0x20, // movdqa xmm0, xmmword ptr [rsp+20h] + 0x66, 0x0F, 0x6F, 0x4C, 0x24, 0x30, // movdqa xmm1, xmmword ptr [rsp+30h] + 0x66, 0x0F, 0x6F, 0x54, 0x24, 0x40, // movdqa xmm2, xmmword ptr [rsp+40h] + 0x66, 0x0F, 0x6F, 0x5C, 0x24, 0x50, // movdqa xmm3, xmmword ptr [rsp+50h] + 0x48, 0x8B, 0x4C, 0x24, 0x70, // mov rcx, qword ptr [rsp+70h] + 0x48, 0x8B, 0x54, 0x24, 0x78, // mov rdx, qword ptr [rsp+78h] + 0x4C, 0x8B, 0x84, 0x24, 0x80, 0, 0, 0, // mov r8, qword ptr [rsp+80h] + 0x4C, 0x8B, 0x8C, 0x24, 0x88, 0, 0, 0, // mov r9, qword ptr [rsp+88h] + 0x48, 0x83, 0xC4, 0x68, // add rsp, 68h + 0xFF, 0xE0, // jmp rax }; static const uint8_t tailMergeUnwindInfoX64[] = { 0x01, // Version=1, Flags=UNW_FLAG_NHANDLER - 0x0a, // Size of prolog - 0x05, // Count of unwind codes + 0x18, // Size of prolog + 0x01, // Count of unwind codes 0x00, // No frame register - 0x0a, 0x82, // Offset 0xa: UWOP_ALLOC_SMALL(0x48) - 0x06, 0x02, // Offset 6: UWOP_ALLOC_SMALL(8) - 0x04, 0x02, // Offset 4: UWOP_ALLOC_SMALL(8) - 0x02, 0x02, // Offset 2: UWOP_ALLOC_SMALL(8) - 0x01, 0x02, // Offset 1: UWOP_ALLOC_SMALL(8) + 0x18, 0xC2, // Offset 0x18: UWOP_ALLOC_SMALL(0x68) 0x00, 0x00 // Padding to align on 32-bits }; @@ -324,30 +320,34 @@ static const uint8_t thunkARM64[] = { }; static const uint8_t tailMergeARM64[] = { - 0xfd, 0x7b, 0xb3, 0xa9, // stp x29, x30, [sp, #-208]! + 0xfd, 0x7b, 0xb2, 0xa9, // stp x29, x30, [sp, #-224]! 0xfd, 0x03, 0x00, 0x91, // mov x29, sp 0xe0, 0x07, 0x01, 0xa9, // stp x0, x1, [sp, #16] 0xe2, 0x0f, 0x02, 0xa9, // stp x2, x3, [sp, #32] 0xe4, 0x17, 0x03, 0xa9, // stp x4, x5, [sp, #48] 0xe6, 0x1f, 0x04, 0xa9, // stp x6, x7, [sp, #64] - 0xe0, 0x87, 0x02, 0xad, // stp q0, q1, [sp, #80] - 0xe2, 0x8f, 0x03, 0xad, // stp q2, q3, [sp, #112] - 0xe4, 0x97, 0x04, 0xad, // stp q4, q5, [sp, #144] - 0xe6, 0x9f, 0x05, 0xad, // stp q6, q7, [sp, #176] + 0xe8, 0x2b, 0x00, 0xf9, // str x8, [sp, #80] + 0xe0, 0x07, 0x03, 0xad, // stp q0, q1, [sp, #96] + 0xe2, 0x0f, 0x04, 0xad, // stp q2, q3, [sp, #128] + 0xe4, 0x17, 0x05, 0xad, // stp q4, q5, [sp, #160] + 0xe6, 0x1f, 0x06, 0xad, // stp q6, q7, [sp, #192] 0xe1, 0x03, 0x11, 0xaa, // mov x1, x17 0x00, 0x00, 0x00, 0x90, // adrp x0, #0 DELAY_IMPORT_DESCRIPTOR 0x00, 0x00, 0x00, 0x91, // add x0, x0, #0 :lo12:DELAY_IMPORT_DESCRIPTOR - 0x00, 0x00, 0x00, 0x94, // bl #0 __delayLoadHelper2 + 0x02, 0x00, 0x00, 0x90, // adrp x2, #0 __delayLoadHelper2 + 0x42, 0x00, 0x00, 0x91, // add x2, x2, #0 :lo12:__delayLoadHelper2 + 0x40, 0x00, 0x3f, 0xd6, // blr x2 0xf0, 0x03, 0x00, 0xaa, // mov x16, x0 - 0xe6, 0x9f, 0x45, 0xad, // ldp q6, q7, [sp, #176] - 0xe4, 0x97, 0x44, 0xad, // ldp q4, q5, [sp, #144] - 0xe2, 0x8f, 0x43, 0xad, // ldp q2, q3, [sp, #112] - 0xe0, 0x87, 0x42, 0xad, // ldp q0, q1, [sp, #80] + 0xe6, 0x1f, 0x46, 0xad, // ldp q6, q7, [sp, #192] + 0xe4, 0x17, 0x45, 0xad, // ldp q4, q5, [sp, #160] + 0xe2, 0x0f, 0x44, 0xad, // ldp q2, q3, [sp, #128] + 0xe0, 0x07, 0x43, 0xad, // ldp q0, q1, [sp, #96] + 0xe8, 0x2b, 0x40, 0xf9, // ldr x8, [sp, #80] 0xe6, 0x1f, 0x44, 0xa9, // ldp x6, x7, [sp, #64] 0xe4, 0x17, 0x43, 0xa9, // ldp x4, x5, [sp, #48] 0xe2, 0x0f, 0x42, 0xa9, // ldp x2, x3, [sp, #32] 0xe0, 0x07, 0x41, 0xa9, // ldp x0, x1, [sp, #16] - 0xfd, 0x7b, 0xcd, 0xa8, // ldp x29, x30, [sp], #208 + 0xfd, 0x7b, 0xce, 0xa8, // ldp x29, x30, [sp], #224 0x00, 0x02, 0x1f, 0xd6, // br x16 }; @@ -378,8 +378,8 @@ public: void writeTo(uint8_t *buf) const override { memcpy(buf, tailMergeX64, sizeof(tailMergeX64)); - write32le(buf + 39, desc->getRVA() - rva - 43); - write32le(buf + 44, helper->getRVA() - rva - 48); + write32le(buf + 54, desc->getRVA() - rva - 58); + write32le(buf + 59, helper->getRVA() - rva - 63); } Chunk *desc = nullptr; @@ -558,10 +558,12 @@ public: void writeTo(uint8_t *buf) const override { memcpy(buf, tailMergeARM64, sizeof(tailMergeARM64)); - applyArm64Addr(buf + 44, desc->getRVA(), rva + 44, 12); - applyArm64Imm(buf + 48, desc->getRVA() & 0xfff, 0); - if (helper) - applyArm64Branch26(buf + 52, helper->getRVA() - rva - 52); + applyArm64Addr(buf + 48, desc->getRVA(), rva + 48, 12); + applyArm64Imm(buf + 52, desc->getRVA() & 0xfff, 0); + if (helper) { + applyArm64Addr(buf + 56, helper->getRVA(), rva + 56, 12); + applyArm64Imm(buf + 60, helper->getRVA() & 0xfff, 0); + } } Chunk *desc = nullptr; diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp index 283aeed1a19c..570b8f9d0590 100644 --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -274,8 +274,13 @@ void LinkerDriver::addBuffer(std::unique_ptr<MemoryBuffer> mb, make<std::unique_ptr<Archive>>(std::move(file)); // take ownership int memberIndex = 0; - for (MemoryBufferRef m : getArchiveMembers(ctx, archive)) - addArchiveBuffer(m, "<whole-archive>", filename, memberIndex++); + for (MemoryBufferRef m : getArchiveMembers(ctx, archive)) { + if (!archive->isThin()) + addArchiveBuffer(m, "<whole-archive>", filename, memberIndex++); + else + addThinArchiveBuffer(m, "<whole-archive>"); + } + return; } addFile(make<ArchiveFile>(ctx, mbref)); @@ -386,6 +391,14 @@ void LinkerDriver::addArchiveBuffer(MemoryBufferRef mb, StringRef symName, Log(ctx) << "Loaded " << obj << " for " << symName; } +void LinkerDriver::addThinArchiveBuffer(MemoryBufferRef mb, StringRef symName) { + // Pass an empty string as the archive name and an offset of 0 so that + // the original filename is used as the buffer identifier. This is + // useful for DTLTO, where having the member identifier be the actual + // path on disk enables distribution of bitcode files during ThinLTO. + addArchiveBuffer(mb, symName, /*parentName=*/"", /*OffsetInArchive=*/0); +} + void LinkerDriver::enqueueArchiveMember(const Archive::Child &c, const Archive::Symbol &sym, StringRef parentName) { @@ -422,11 +435,8 @@ void LinkerDriver::enqueueArchiveMember(const Archive::Child &c, reportBufferError(errorCodeToError(mbOrErr.second), childName); llvm::TimeTraceScope timeScope("Archive: ", mbOrErr.first->getBufferIdentifier()); - // Pass empty string as archive name so that the original filename is - // used as the buffer identifier. - ctx.driver.addArchiveBuffer(takeBuffer(std::move(mbOrErr.first)), - toCOFFString(ctx, sym), "", - /*OffsetInArchive=*/0); + ctx.driver.addThinArchiveBuffer(takeBuffer(std::move(mbOrErr.first)), + toCOFFString(ctx, sym)); }); } @@ -1643,8 +1653,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { config->warnLocallyDefinedImported = false; else if (s == "longsections") config->warnLongSectionNames = false; - else if (s == "exporteddllmain") - config->warnExportedDllMain = false; + else if (s == "importeddllmain") + config->warnImportedDllMain = false; // Other warning numbers are ignored. } } @@ -2088,6 +2098,23 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { Fatal(ctx) << "/manifestinput: requires /manifest:embed"; } + // Handle /thinlto-distributor:<path> + config->dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor); + + // Handle /thinlto-distributor-arg:<arg> + for (auto *arg : args.filtered(OPT_thinlto_distributor_arg)) + config->dtltoDistributorArgs.push_back(arg->getValue()); + + // Handle /thinlto-remote-compiler:<path> + config->dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler); + if (!config->dtltoDistributor.empty() && config->dtltoCompiler.empty()) + Err(ctx) << "A value must be specified for /thinlto-remote-compiler if " + "/thinlto-distributor is specified."; + + // Handle /thinlto-remote-compiler-arg:<arg> + for (auto *arg : args.filtered(OPT_thinlto_compiler_arg)) + config->dtltoCompilerArgs.push_back(arg->getValue()); + // Handle /dwodir config->dwoDir = args.getLastArgValue(OPT_dwodir); @@ -2527,28 +2554,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { e.symbolName = symtab.mangleMaybe(e.sym); } - // Add weak aliases. Weak aliases is a mechanism to give remaining - // undefined symbols final chance to be resolved successfully. - for (auto pair : symtab.alternateNames) { - StringRef from = pair.first; - StringRef to = pair.second; - Symbol *sym = symtab.find(from); - if (!sym) - continue; - if (auto *u = dyn_cast<Undefined>(sym)) { - if (u->weakAlias) { - // On ARM64EC, anti-dependency aliases are treated as undefined - // symbols unless a demangled symbol aliases a defined one, which - // is part of the implementation. - if (!symtab.isEC() || !u->isAntiDep) - continue; - if (!isa<Undefined>(u->weakAlias) && - !isArm64ECMangledFunctionName(u->getName())) - continue; - } - u->setWeakAlias(symtab.addUndefined(to)); - } - } + symtab.resolveAlternateNames(); }); ctx.forEachActiveSymtab([&](SymbolTable &symtab) { diff --git a/lld/COFF/Driver.h b/lld/COFF/Driver.h index 14c97a98875b..5a9bd5c6d968 100644 --- a/lld/COFF/Driver.h +++ b/lld/COFF/Driver.h @@ -173,6 +173,7 @@ private: bool lazy); void addArchiveBuffer(MemoryBufferRef mbref, StringRef symName, StringRef parentName, uint64_t offsetInArchive); + void addThinArchiveBuffer(MemoryBufferRef mbref, StringRef symName); void enqueueTask(std::function<void()> task); bool run(); diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp index d8b41c7f4540..5ef41c4c0a08 100644 --- a/lld/COFF/DriverUtils.cpp +++ b/lld/COFF/DriverUtils.cpp @@ -387,7 +387,7 @@ std::string LinkerDriver::createDefaultXml() { << "<assembly xmlns=\"urn:schemas-microsoft-com:asm.v1\"\n" << " manifestVersion=\"1.0\">\n"; if (ctx.config.manifestUAC) { - os << " <trustInfo>\n" + os << " <trustInfo xmlns=\"urn:schemas-microsoft-com:asm.v3\">\n" << " <security>\n" << " <requestedPrivileges>\n" << " <requestedExecutionLevel level=" << ctx.config.manifestLevel diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp index 0b7dbea8cdd9..2a6b63cbacca 100644 --- a/lld/COFF/InputFiles.cpp +++ b/lld/COFF/InputFiles.cpp @@ -117,8 +117,6 @@ static coff_symbol_generic *cloneSymbol(COFFSymbolRef sym) { // Skip importing DllMain thunks from import libraries. static bool fixupDllMain(COFFLinkerContext &ctx, llvm::object::Archive *file, const Archive::Symbol &sym, bool &skipDllMain) { - if (skipDllMain) - return true; const Archive::Child &c = CHECK(sym.getMember(), file->getFileName() + ": could not get the member for symbol " + @@ -128,13 +126,13 @@ static bool fixupDllMain(COFFLinkerContext &ctx, llvm::object::Archive *file, file->getFileName() + ": could not get the buffer for a child buffer of the archive"); if (identify_magic(mb.getBuffer()) == file_magic::coff_import_library) { - if (ctx.config.warnExportedDllMain) { + if (ctx.config.warnImportedDllMain) { // We won't place DllMain symbols in the symbol table if they are // coming from a import library. This message can be ignored with the flag - // '/ignore:exporteddllmain' + // '/ignore:importeddllmain' Warn(ctx) << file->getFileName() - << ": skipping exported DllMain symbol [exporteddllmain]\nNOTE: this " + << ": skipping imported DllMain symbol [importeddllmain]\nNOTE: this " "might be a mistake when the DLL/library was produced."; } skipDllMain = true; @@ -204,14 +202,24 @@ void ArchiveFile::parse() { } } - // Read the symbol table to construct Lazy objects. bool skipDllMain = false; + StringRef mangledDllMain, impMangledDllMain; + + // The calls below will fail if we haven't set the machine type yet. Instead + // of failing, it is preferable to skip this "imported DllMain" check if we + // don't know the machine type at this point. + if (!file->isEmpty() && ctx.config.machine != IMAGE_FILE_MACHINE_UNKNOWN) { + mangledDllMain = archiveSymtab->mangle("DllMain"); + impMangledDllMain = uniqueSaver().save("__imp_" + mangledDllMain); + } + + // Read the symbol table to construct Lazy objects. for (const Archive::Symbol &sym : file->symbols()) { - // If the DllMain symbol was exported by mistake, skip importing it - // otherwise we might end up with a import thunk in the final binary which - // is wrong. - if (sym.getName() == "__imp_DllMain" || sym.getName() == "DllMain") { - if (fixupDllMain(ctx, file.get(), sym, skipDllMain)) + // If an import library provides the DllMain symbol, skip importing it, as + // we should be using our own DllMain, not another DLL's DllMain. + if (!mangledDllMain.empty() && (sym.getName() == mangledDllMain || + sym.getName() == impMangledDllMain)) { + if (skipDllMain || fixupDllMain(ctx, file.get(), sym, skipDllMain)) continue; } archiveSymtab->addLazyArchive(this, sym); diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 2a4d07cc2d01..1050874a1b10 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -110,7 +110,16 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { // Initialize ltoObj. lto::ThinBackend backend; - if (ctx.config.thinLTOIndexOnly) { + if (!ctx.config.dtltoDistributor.empty()) { + backend = lto::createOutOfProcessThinBackend( + llvm::hardware_concurrency(ctx.config.thinLTOJobs), + /*OnWrite=*/nullptr, + /*ShouldEmitIndexFiles=*/false, + /*ShouldEmitImportFiles=*/false, ctx.config.outputFile, + ctx.config.dtltoDistributor, ctx.config.dtltoDistributorArgs, + ctx.config.dtltoCompiler, ctx.config.dtltoCompilerArgs, + !ctx.config.saveTempsArgs.empty()); + } else if (ctx.config.thinLTOIndexOnly) { auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); }; backend = lto::createWriteIndexesThinBackend( llvm::hardware_concurrency(ctx.config.thinLTOJobs), diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td index a887d7d351e1..2a82fb5cd884 100644 --- a/lld/COFF/Options.td +++ b/lld/COFF/Options.td @@ -270,6 +270,17 @@ def thinlto_object_suffix_replace : P< def thinlto_prefix_replace: P< "thinlto-prefix-replace", "'old;new' replace old prefix with new prefix in ThinLTO outputs">; +def thinlto_distributor : P<"thinlto-distributor", + "Distributor to use for ThinLTO backend compilations. If specified, ThinLTO " + "backend compilations will be distributed">; +def thinlto_distributor_arg : P<"thinlto-distributor-arg", + "Arguments to pass to the ThinLTO distributor">; +def thinlto_compiler : P<"thinlto-remote-compiler", + "Compiler for the ThinLTO distributor to invoke for ThinLTO backend " + "compilations">; +def thinlto_compiler_arg : P<"thinlto-remote-compiler-arg", + "Compiler arguments for the ThinLTO distributor to pass for ThinLTO backend " + "compilations">; def lto_obj_path : P< "lto-obj-path", "output native object for merged LTO unit to this path">; diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp index a54ea403ba2e..94eeae279797 100644 --- a/lld/COFF/PDB.cpp +++ b/lld/COFF/PDB.cpp @@ -1135,9 +1135,12 @@ static pdb::BulkPublic createPublic(COFFLinkerContext &ctx, Defined *def) { pub.setFlags(flags); OutputSection *os = ctx.getOutputSection(def->getChunk()); - assert(os && "all publics should be in final image"); - pub.Offset = def->getRVA() - os->getRVA(); - pub.Segment = os->sectionIndex; + assert((os || !def->getChunk()->getSize()) && + "all publics should be in final image"); + if (os) { + pub.Offset = def->getRVA() - os->getRVA(); + pub.Segment = os->sectionIndex; + } return pub; } diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp index 0062df5820e6..d15e0c24410c 100644 --- a/lld/COFF/SymbolTable.cpp +++ b/lld/COFF/SymbolTable.cpp @@ -1344,6 +1344,44 @@ void SymbolTable::parseAlternateName(StringRef s) { alternateNames.insert(it, std::make_pair(from, to)); } +void SymbolTable::resolveAlternateNames() { + // Add weak aliases. Weak aliases is a mechanism to give remaining + // undefined symbols final chance to be resolved successfully. + for (auto pair : alternateNames) { + StringRef from = pair.first; + StringRef to = pair.second; + Symbol *sym = find(from); + if (!sym) + continue; + if (auto *u = dyn_cast<Undefined>(sym)) { + if (u->weakAlias) { + // On ARM64EC, anti-dependency aliases are treated as undefined + // symbols unless a demangled symbol aliases a defined one, which + // is part of the implementation. + if (!isEC() || !u->isAntiDep) + continue; + if (!isa<Undefined>(u->weakAlias) && + !isArm64ECMangledFunctionName(u->getName())) + continue; + } + + // Check if the destination symbol is defined. If not, skip it. + // It may still be resolved later if more input files are added. + // Also skip anti-dependency targets, as they can't be chained anyway. + Symbol *toSym = find(to); + if (!toSym) + continue; + auto toUndef = dyn_cast<Undefined>(toSym); + if (toUndef && (!toUndef->weakAlias || toUndef->isAntiDep)) + continue; + toSym->isUsedInRegularObj = true; + if (toSym->isLazy()) + forceLazy(toSym); + u->setWeakAlias(toSym); + } + } +} + // Parses /aligncomm option argument. void SymbolTable::parseAligncomm(StringRef s) { auto [name, align] = s.split(','); diff --git a/lld/COFF/SymbolTable.h b/lld/COFF/SymbolTable.h index 15e2644a6f51..7eb067640dc8 100644 --- a/lld/COFF/SymbolTable.h +++ b/lld/COFF/SymbolTable.h @@ -69,6 +69,9 @@ public: // symbols and warn about imported local symbols. void resolveRemainingUndefines(); + // Try to resolve undefined symbols with alternate names. + void resolveAlternateNames(); + // Load lazy objects that are needed for MinGW automatic import and for // doing stdcall fixups. void loadMinGWSymbols(); diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp index 076561807af4..ef9d051bf976 100644 --- a/lld/COFF/Writer.cpp +++ b/lld/COFF/Writer.cpp @@ -1553,7 +1553,7 @@ void Writer::createSymbolAndStringTable() { dthunk->wrappedSym->writtenToSymtab = true; if (std::optional<coff_symbol16> sym = createSymbol(dthunk->wrappedSym)) { - if (d->getName().size() > COFF::NameSize) + if (dthunk->wrappedSym->getName().size() > COFF::NameSize) longNameSymbols.emplace_back(outputSymtab.size(), dthunk->wrappedSym->getName()); outputSymtab.push_back(*sym); diff --git a/lld/ELF/Arch/Hexagon.cpp b/lld/ELF/Arch/Hexagon.cpp index 479131a24dcf..9b33e78731c9 100644 --- a/lld/ELF/Arch/Hexagon.cpp +++ b/lld/ELF/Arch/Hexagon.cpp @@ -11,6 +11,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" +#include "Thunks.h" #include "lld/Common/ErrorHandler.h" #include "llvm/ADT/SmallVector.h" #include "llvm/BinaryFormat/ELF.h" @@ -36,6 +37,10 @@ public: const uint8_t *loc) const override; RelType getDynRel(RelType type) const override; int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; + bool needsThunk(RelExpr expr, RelType type, const InputFile *file, + uint64_t branchAddr, const Symbol &s, + int64_t a) const override; + bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; void relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const override; void writePltHeader(uint8_t *buf) const override; @@ -63,6 +68,8 @@ Hexagon::Hexagon(Ctx &ctx) : TargetInfo(ctx) { tlsGotRel = R_HEX_TPREL_32; tlsModuleIndexRel = R_HEX_DTPMOD_32; tlsOffsetRel = R_HEX_DTPREL_32; + + needsThunks = true; } uint32_t Hexagon::calcEFlags() const { @@ -258,6 +265,46 @@ static uint32_t findMaskR16(Ctx &ctx, uint32_t insn) { static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); } +bool Hexagon::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { + int64_t offset = dst - src; + switch (type) { + case llvm::ELF::R_HEX_B22_PCREL: + case llvm::ELF::R_HEX_PLT_B22_PCREL: + case llvm::ELF::R_HEX_GD_PLT_B22_PCREL: + case llvm::ELF::R_HEX_LD_PLT_B22_PCREL: + return llvm::isInt<22>(offset >> 2); + case llvm::ELF::R_HEX_B15_PCREL: + return llvm::isInt<15>(offset >> 2); + break; + case llvm::ELF::R_HEX_B13_PCREL: + return llvm::isInt<13>(offset >> 2); + break; + case llvm::ELF::R_HEX_B9_PCREL: + return llvm::isInt<9>(offset >> 2); + default: + return true; + } + llvm_unreachable("unsupported relocation"); +} + +bool Hexagon::needsThunk(RelExpr expr, RelType type, const InputFile *file, + uint64_t branchAddr, const Symbol &s, + int64_t a) const { + // Only check branch range for supported branch relocation types + switch (type) { + case R_HEX_B22_PCREL: + case R_HEX_PLT_B22_PCREL: + case R_HEX_GD_PLT_B22_PCREL: + case R_HEX_LD_PLT_B22_PCREL: + case R_HEX_B15_PCREL: + case R_HEX_B13_PCREL: + case R_HEX_B9_PCREL: + return !ctx.target->inBranchRange(type, branchAddr, s.getVA(ctx, a)); + default: + return false; + } +} + void Hexagon::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { switch (rel.type) { diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp index fe804cbb0e69..8802c8c2e7f0 100644 --- a/lld/ELF/Arch/LoongArch.cpp +++ b/lld/ELF/Arch/LoongArch.cpp @@ -46,6 +46,8 @@ public: private: void tlsdescToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const; void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + bool tryGotToPCRel(uint8_t *loc, const Relocation &rHi20, + const Relocation &rLo12, uint64_t secAddr) const; }; } // end anonymous namespace @@ -809,10 +811,13 @@ static void relaxPCHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i, // address. // Meanwhile skip undefined, preemptible and STT_GNU_IFUNC symbols, because // these symbols may be resolve in runtime. + // Moreover, relaxation can only occur if the addends of both relocations are + // zero for GOT references. if (rHi20.type == R_LARCH_GOT_PC_HI20 && - (!rHi20.sym->isDefined() || rHi20.sym->isPreemptible || - rHi20.sym->isGnuIFunc() || - (ctx.arg.isPic && !cast<Defined>(*rHi20.sym).section))) + (!rHi20.sym || rHi20.sym != rLo12.sym || !rHi20.sym->isDefined() || + rHi20.sym->isPreemptible || rHi20.sym->isGnuIFunc() || + (ctx.arg.isPic && !cast<Defined>(*rHi20.sym).section) || + rHi20.addend != 0 || rLo12.addend != 0)) return; uint64_t dest = 0; @@ -966,10 +971,16 @@ static bool relax(Ctx &ctx, InputSection &sec) { case R_LARCH_GOT_PC_HI20: case R_LARCH_TLS_GD_PC_HI20: case R_LARCH_TLS_LD_PC_HI20: - case R_LARCH_TLS_DESC_PC_HI20: // The overflow check for i+2 will be carried out in isPairRelaxable. - if (r.expr != RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC && - r.expr != R_RELAX_TLS_GD_TO_LE && isPairRelaxable(relocs, i)) + if (isPairRelaxable(relocs, i)) + relaxPCHi20Lo12(ctx, sec, i, loc, r, relocs[i + 2], remove); + break; + case R_LARCH_TLS_DESC_PC_HI20: + if (r.expr == RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC || + r.expr == R_RELAX_TLS_GD_TO_LE) { + if (relaxable(relocs, i)) + remove = 4; + } else if (isPairRelaxable(relocs, i)) relaxPCHi20Lo12(ctx, sec, i, loc, r, relocs[i + 2], remove); break; case R_LARCH_CALL36: @@ -987,6 +998,17 @@ static bool relax(Ctx &ctx, InputSection &sec) { isUInt<12>(r.sym->getVA(ctx, r.addend))) remove = 4; break; + case R_LARCH_TLS_DESC_PC_LO12: + if (relaxable(relocs, i) && + (r.expr == RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC || + r.expr == R_RELAX_TLS_GD_TO_LE)) + remove = 4; + break; + case R_LARCH_TLS_DESC_LD: + if (relaxable(relocs, i) && r.expr == R_RELAX_TLS_GD_TO_LE && + isUInt<12>(r.sym->getVA(ctx, r.addend))) + remove = 4; + break; } // For all anchors whose offsets are <= r.offset, they are preceded by @@ -1135,6 +1157,78 @@ void LoongArch::tlsdescToLe(uint8_t *loc, const Relocation &rel, } } +// Try GOT indirection to PC relative optimization. +// From: +// * pcalau12i $a0, %got_pc_hi20(sym_got) +// * ld.w/d $a0, $a0, %got_pc_lo12(sym_got) +// To: +// * pcalau12i $a0, %pc_hi20(sym) +// * addi.w/d $a0, $a0, %pc_lo12(sym) +// +// Note: Althouth the optimization has been performed, the GOT entries still +// exists, similarly to AArch64. Eliminating the entries will increase code +// complexity. +bool LoongArch::tryGotToPCRel(uint8_t *loc, const Relocation &rHi20, + const Relocation &rLo12, uint64_t secAddr) const { + // Check if the relocations apply to consecutive instructions. + if (rHi20.offset + 4 != rLo12.offset) + return false; + + // Check if the relocations reference the same symbol and skip undefined, + // preemptible and STT_GNU_IFUNC symbols. + if (!rHi20.sym || rHi20.sym != rLo12.sym || !rHi20.sym->isDefined() || + rHi20.sym->isPreemptible || rHi20.sym->isGnuIFunc()) + return false; + + // GOT references to absolute symbols can't be relaxed to use PCALAU12I/ADDI + // in position-independent code because these instructions produce a relative + // address. + if ((ctx.arg.isPic && !cast<Defined>(*rHi20.sym).section)) + return false; + + // Check if the addends of the both relocations are zero. + if (rHi20.addend != 0 || rLo12.addend != 0) + return false; + + const uint32_t currInsn = read32le(loc); + const uint32_t nextInsn = read32le(loc + 4); + const uint32_t ldOpcode = ctx.arg.is64 ? LD_D : LD_W; + // Check if the first instruction is PCALAU12I and the second instruction is + // LD. + if ((currInsn & 0xfe000000) != PCALAU12I || + (nextInsn & 0xffc00000) != ldOpcode) + return false; + + // Check if use the same register. + if (getD5(currInsn) != getJ5(nextInsn) || getJ5(nextInsn) != getD5(nextInsn)) + return false; + + Symbol &sym = *rHi20.sym; + uint64_t symLocal = sym.getVA(ctx); + const int64_t displace = symLocal - getLoongArchPage(secAddr + rHi20.offset); + // Check if the symbol address is in + // [(PC & ~0xfff) - 2GiB - 0x800, (PC & ~0xfff) + 2GiB - 0x800). + const int64_t underflow = -0x80000000LL - 0x800; + const int64_t overflow = 0x80000000LL - 0x800; + if (!(displace >= underflow && displace < overflow)) + return false; + + Relocation newRHi20 = {RE_LOONGARCH_PAGE_PC, R_LARCH_PCALA_HI20, rHi20.offset, + rHi20.addend, &sym}; + Relocation newRLo12 = {R_ABS, R_LARCH_PCALA_LO12, rLo12.offset, rLo12.addend, + &sym}; + uint64_t pageDelta = + getLoongArchPageDelta(symLocal, secAddr + rHi20.offset, rHi20.type); + // pcalau12i $a0, %pc_hi20 + write32le(loc, insn(PCALAU12I, getD5(currInsn), 0, 0)); + relocate(loc, newRHi20, pageDelta); + // addi.w/d $a0, $a0, %pc_lo12 + write32le(loc + 4, insn(ctx.arg.is64 ? ADDI_D : ADDI_W, getD5(nextInsn), + getJ5(nextInsn), 0)); + relocate(loc + 4, newRLo12, SignExtend64(symLocal, 64)); + return true; +} + // During TLSDESC GD_TO_IE, the converted code sequence always includes an // instruction related to the Lo12 relocation (ld.[wd]). To obtain correct val // in `getRelocTargetVA`, expr of this instruction should be adjusted to @@ -1152,6 +1246,30 @@ RelExpr LoongArch::adjustTlsExpr(RelType type, RelExpr expr) const { return expr; } +static bool pairForGotRels(ArrayRef<Relocation> relocs) { + // Check if R_LARCH_GOT_PC_HI20 and R_LARCH_GOT_PC_LO12 always appear in + // pairs. + size_t i = 0; + const size_t size = relocs.size(); + for (; i != size; ++i) { + if (relocs[i].type == R_LARCH_GOT_PC_HI20) { + if (i + 1 < size && relocs[i + 1].type == R_LARCH_GOT_PC_LO12) { + ++i; + continue; + } + if (relaxable(relocs, i) && i + 2 < size && + relocs[i + 2].type == R_LARCH_GOT_PC_LO12) { + i += 2; + continue; + } + break; + } else if (relocs[i].type == R_LARCH_GOT_PC_LO12) { + break; + } + } + return i == size; +} + void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { const unsigned bits = ctx.arg.is64 ? 64 : 32; uint64_t secAddr = sec.getOutputSection()->addr; @@ -1161,6 +1279,7 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { secAddr += ehIn->getParent()->outSecOff; bool isExtreme = false, isRelax = false; const MutableArrayRef<Relocation> relocs = sec.relocs(); + const bool isPairForGotRels = pairForGotRels(relocs); for (size_t i = 0, size = relocs.size(); i != size; ++i) { Relocation &rel = relocs[i]; uint8_t *loc = buf + rel.offset; @@ -1216,6 +1335,10 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { bits); relocateNoSym(loc, rel.type, val); } else { + isRelax = relaxable(relocs, i); + if (isRelax && (rel.type == R_LARCH_TLS_DESC_PC_HI20 || + rel.type == R_LARCH_TLS_DESC_PC_LO12)) + continue; tlsdescToIe(loc, rel, val); } continue; @@ -1232,9 +1355,32 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { bits); relocateNoSym(loc, rel.type, val); } else { + isRelax = relaxable(relocs, i); + if (isRelax && (rel.type == R_LARCH_TLS_DESC_PC_HI20 || + rel.type == R_LARCH_TLS_DESC_PC_LO12 || + (rel.type == R_LARCH_TLS_DESC_LD && isUInt<12>(val)))) + continue; tlsdescToLe(loc, rel, val); } continue; + case RE_LOONGARCH_GOT_PAGE_PC: + // In LoongArch, we try GOT indirection to PC relative optimization in + // normal or medium code model, whether or not with R_LARCH_RELAX + // relocation. Moreover, if the original code sequence can be relaxed to a + // single instruction `pcaddi`, the first instruction will be removed and + // it will not reach here. + if (isPairForGotRels && rel.type == R_LARCH_GOT_PC_HI20) { + bool isRelax = relaxable(relocs, i); + const Relocation lo12Rel = isRelax ? relocs[i + 2] : relocs[i + 1]; + if (lo12Rel.type == R_LARCH_GOT_PC_LO12 && + tryGotToPCRel(loc, rel, lo12Rel, secAddr)) { + // isRelax: skip relocations R_LARCH_RELAX, R_LARCH_GOT_PC_LO12 + // !isRelax: skip relocation R_LARCH_GOT_PC_LO12 + i += isRelax ? 2 : 1; + continue; + } + } + break; default: break; } diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 71e72e7184b9..62b54f8217bc 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/STLExtras.h" #include "llvm/LTO/LTO.h" +#include "llvm/Object/Archive.h" #include "llvm/Object/IRObjectFile.h" #include "llvm/Support/ARMAttributeParser.h" #include "llvm/Support/ARMBuildAttributes.h" @@ -1753,6 +1754,39 @@ static uint8_t getOsAbi(const Triple &t) { } } +// For DTLTO, bitcode member names must be valid paths to files on disk. +// For thin archives, resolve `memberPath` relative to the archive's location. +// Returns true if adjusted; false otherwise. Non-thin archives are unsupported. +static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath, + std::string &memberPath) { + assert(!archivePath.empty()); + + if (ctx.arg.dtltoDistributor.empty()) + return false; + + // Read the archive header to determine if it's a thin archive. + auto bufferOrErr = + MemoryBuffer::getFileSlice(archivePath, sizeof(ThinArchiveMagic) - 1, 0); + if (std::error_code ec = bufferOrErr.getError()) { + ErrAlways(ctx) << "cannot open " << archivePath << ": " << ec.message(); + return false; + } + + if (!bufferOrErr->get()->getBuffer().starts_with(ThinArchiveMagic)) + return false; + + SmallString<128> resolvedPath; + if (path::is_relative(memberPath)) { + resolvedPath = path::parent_path(archivePath); + path::append(resolvedPath, memberPath); + } else + resolvedPath = memberPath; + + path::remove_dots(resolvedPath, /*remove_dot_dot=*/true); + memberPath = resolvedPath.str(); + return true; +} + BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, uint64_t offsetInArchive, bool lazy) : InputFile(ctx, BitcodeKind, mb) { @@ -1763,17 +1797,22 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, if (ctx.arg.thinLTOIndexOnly) path = replaceThinLTOSuffix(ctx, mb.getBufferIdentifier()); - // ThinLTO assumes that all MemoryBufferRefs given to it have a unique - // name. If two archives define two members with the same name, this - // causes a collision which result in only one of the objects being taken - // into consideration at LTO time (which very likely causes undefined - // symbols later in the link stage). So we append file offset to make - // filename unique. StringSaver &ss = ctx.saver; - StringRef name = archiveName.empty() - ? ss.save(path) - : ss.save(archiveName + "(" + path::filename(path) + - " at " + utostr(offsetInArchive) + ")"); + StringRef name; + if (archiveName.empty() || + dtltoAdjustMemberPathIfThinArchive(ctx, archiveName, path)) { + name = ss.save(path); + } else { + // ThinLTO assumes that all MemoryBufferRefs given to it have a unique + // name. If two archives define two members with the same name, this + // causes a collision which result in only one of the objects being taken + // into consideration at LTO time (which very likely causes undefined + // symbols later in the link stage). So we append file offset to make + // filename unique. + name = ss.save(archiveName + "(" + path::filename(path) + " at " + + utostr(offsetInArchive) + ")"); + } + MemoryBufferRef mbref(mb.getBuffer(), name); obj = CHECK2(lto::InputFile::create(mbref), this); diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp index a5d08f4979da..b55c19933a13 100644 --- a/lld/ELF/LinkerScript.cpp +++ b/lld/ELF/LinkerScript.cpp @@ -1021,10 +1021,6 @@ void LinkerScript::addOrphanSections() { } }; - // For further --emit-reloc handling code we need target output section - // to be created before we create relocation output section, so we want - // to create target sections first. We do not want priority handling - // for synthetic sections because them are special. size_t n = 0; for (InputSectionBase *isec : ctx.inputSections) { // Process InputSection and MergeInputSection. @@ -1037,10 +1033,18 @@ void LinkerScript::addOrphanSections() { if (ctx.arg.relocatable && (isec->flags & SHF_LINK_ORDER)) continue; - if (auto *sec = dyn_cast<InputSection>(isec)) - if (InputSectionBase *rel = sec->getRelocatedSection()) - if (auto *relIS = dyn_cast_or_null<InputSectionBase>(rel->parent)) - add(relIS); + if (auto *sec = dyn_cast<InputSection>(isec)) { + if (InputSectionBase *relocated = sec->getRelocatedSection()) { + // For --emit-relocs and -r, ensure the output section for .text.foo + // is created before the output section for .rela.text.foo. + add(relocated); + // EhInputSection sections are not added to ctx.inputSections. If we see + // .rela.eh_frame, ensure the output section for the synthetic + // EhFrameSection is created first. + if (auto *p = dyn_cast_or_null<InputSectionBase>(relocated->parent)) + add(p); + } + } add(isec); if (ctx.arg.relocatable) for (InputSectionBase *depSec : isec->dependentSections) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index cebd564036b2..608cdd0d2666 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -2139,19 +2139,45 @@ void ThunkCreator::mergeThunks(ArrayRef<OutputSection *> outputSections) { }); } -static int64_t getPCBias(Ctx &ctx, RelType type) { - if (ctx.arg.emachine != EM_ARM) - return 0; - switch (type) { - case R_ARM_THM_JUMP19: - case R_ARM_THM_JUMP24: - case R_ARM_THM_CALL: - return 4; - default: - return 8; +constexpr uint32_t HEXAGON_MASK_END_PACKET = 3 << 14; +constexpr uint32_t HEXAGON_END_OF_PACKET = 3 << 14; +constexpr uint32_t HEXAGON_END_OF_DUPLEX = 0 << 14; + +// Return the distance between the packet start and the instruction in the +// relocation. +static int getHexagonPacketOffset(const InputSection &isec, + const Relocation &rel) { + const ArrayRef<uint8_t> data = isec.content(); + + // Search back as many as 3 instructions. + for (unsigned i = 0;; i++) { + if (i == 3 || rel.offset < (i + 1) * 4) + return i * 4; + uint32_t instWord = + read32(isec.getCtx(), data.data() + (rel.offset - (i + 1) * 4)); + if (((instWord & HEXAGON_MASK_END_PACKET) == HEXAGON_END_OF_PACKET) || + ((instWord & HEXAGON_MASK_END_PACKET) == HEXAGON_END_OF_DUPLEX)) + return i * 4; } } +static int64_t getPCBias(Ctx &ctx, const InputSection &isec, + const Relocation &rel) { + if (ctx.arg.emachine == EM_ARM) { + switch (rel.type) { + case R_ARM_THM_JUMP19: + case R_ARM_THM_JUMP24: + case R_ARM_THM_CALL: + return 4; + default: + return 8; + } + } + if (ctx.arg.emachine == EM_HEXAGON) + return -getHexagonPacketOffset(isec, rel); + return 0; +} + // Find or create a ThunkSection within the InputSectionDescription (ISD) that // is in range of Src. An ISD maps to a range of InputSections described by a // linker script section pattern such as { .text .text.* }. @@ -2161,7 +2187,7 @@ ThunkSection *ThunkCreator::getISDThunkSec(OutputSection *os, const Relocation &rel, uint64_t src) { // See the comment in getThunk for -pcBias below. - const int64_t pcBias = getPCBias(ctx, rel.type); + const int64_t pcBias = getPCBias(ctx, *isec, rel); for (std::pair<ThunkSection *, uint32_t> tp : isd->thunkSections) { ThunkSection *ts = tp.first; uint64_t tsBase = os->addr + ts->outSecOff - pcBias; @@ -2322,7 +2348,7 @@ std::pair<Thunk *, bool> ThunkCreator::getThunk(InputSection *isec, // out in the relocation addend. We compensate for the PC bias so that // an Arm and Thumb relocation to the same destination get the same keyAddend, // which is usually 0. - const int64_t pcBias = getPCBias(ctx, rel.type); + const int64_t pcBias = getPCBias(ctx, *isec, rel); const int64_t keyAddend = rel.addend + pcBias; // We use a ((section, offset), addend) pair to find the thunk position if @@ -2481,7 +2507,7 @@ bool ThunkCreator::createThunks(uint32_t pass, // STT_SECTION + non-zero addend, clear the addend after // redirection. if (ctx.arg.emachine != EM_MIPS) - rel.addend = -getPCBias(ctx, rel.type); + rel.addend = -getPCBias(ctx, *isec, rel); } for (auto &p : isd->thunkSections) @@ -2525,7 +2551,8 @@ void elf::hexagonTLSSymbolUpdate(Ctx &ctx) { for (Relocation &rel : isec->relocs()) if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) { if (needEntry) { - sym->allocateAux(ctx); + if (sym->auxIdx == 0) + sym->allocateAux(ctx); addPltEntry(ctx, *ctx.in.plt, *ctx.in.gotPlt, *ctx.in.relaPlt, ctx.target->pltRel, *sym); needEntry = false; diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp index c26ba76bccb7..65d0f094c43c 100644 --- a/lld/ELF/Thunks.cpp +++ b/lld/ELF/Thunks.cpp @@ -415,6 +415,22 @@ public: void addSymbols(ThunkSection &isec) override; }; +// Hexagon CPUs need thunks for R_HEX_B{9,1{3,5},22}_PCREL, +// R_HEX_{,GD_}PLT_B22_PCREL when their destination is out of +// range. +class HexagonThunk : public Thunk { +public: + HexagonThunk(Ctx &ctx, const InputSection &isec, Relocation &rel, + Symbol &dest) + : Thunk(ctx, dest, 0), relOffset(rel.offset) { + alignment = 4; + } + uint32_t relOffset; + uint32_t size() override { return ctx.arg.isPic ? 12 : 8; } + void writeTo(uint8_t *buf) override; + void addSymbols(ThunkSection &isec) override; +}; + // MIPS LA25 thunk class MipsThunk final : public Thunk { public: @@ -1519,6 +1535,39 @@ bool PPC64LongBranchThunk::isCompatibleWith(const InputSection &isec, return rel.type == R_PPC64_REL24 || rel.type == R_PPC64_REL14; } +// Hexagon Target Thunks +static uint64_t getHexagonThunkDestVA(Ctx &ctx, const Symbol &s, int64_t a) { + uint64_t v = s.isInPlt(ctx) ? s.getPltVA(ctx) : s.getVA(ctx, a); + return SignExtend64<32>(v); +} + +void HexagonThunk::writeTo(uint8_t *buf) { + uint64_t s = getHexagonThunkDestVA(ctx, destination, addend); + uint64_t p = getThunkTargetSym()->getVA(ctx); + + if (ctx.arg.isPic) { + write32(ctx, buf + 0, 0x00004000); // { immext(#0) + ctx.target->relocateNoSym(buf, R_HEX_B32_PCREL_X, s - p); + write32(ctx, buf + 4, 0x6a49c00e); // r14 = add(pc,##0) } + ctx.target->relocateNoSym(buf + 4, R_HEX_6_PCREL_X, s - p); + + write32(ctx, buf + 8, 0x528ec000); // { jumpr r14 } + } else { + write32(ctx, buf + 0, 0x00004000); // { immext + ctx.target->relocateNoSym(buf, R_HEX_B32_PCREL_X, s - p); + write32(ctx, buf + 4, 0x5800c000); // jump <> } + ctx.target->relocateNoSym(buf + 4, R_HEX_B22_PCREL_X, s - p); + } +} +void HexagonThunk::addSymbols(ThunkSection &isec) { + Symbol *enclosing = isec.getEnclosingSymbol(relOffset); + StringRef src = enclosing ? enclosing->getName() : isec.name; + + addSymbol( + saver().save("__hexagon_thunk_" + destination.getName() + "_from_" + src), + STT_FUNC, 0, isec); +} + Thunk::Thunk(Ctx &ctx, Symbol &d, int64_t a) : ctx(ctx), destination(d), addend(a), offset(0) { destination.thunkAccessed = true; @@ -1692,6 +1741,24 @@ static std::unique_ptr<Thunk> addThunkAVR(Ctx &ctx, RelType type, Symbol &s, } } +static std::unique_ptr<Thunk> addThunkHexagon(Ctx &ctx, + const InputSection &isec, + Relocation &rel, Symbol &s) { + switch (rel.type) { + case R_HEX_B9_PCREL: + case R_HEX_B13_PCREL: + case R_HEX_B15_PCREL: + case R_HEX_B22_PCREL: + case R_HEX_PLT_B22_PCREL: + case R_HEX_GD_PLT_B22_PCREL: + return std::make_unique<HexagonThunk>(ctx, isec, rel, s); + default: + Fatal(ctx) << "unrecognized relocation " << rel.type << " to " << &s + << " for hexagon target"; + llvm_unreachable(""); + } +} + static std::unique_ptr<Thunk> addThunkMips(Ctx &ctx, RelType type, Symbol &s) { if ((s.stOther & STO_MIPS_MICROMIPS) && isMipsR6(ctx)) return std::make_unique<MicroMipsR6Thunk>(ctx, s); @@ -1761,8 +1828,11 @@ std::unique_ptr<Thunk> elf::addThunk(Ctx &ctx, const InputSection &isec, return addThunkPPC32(ctx, isec, rel, s); case EM_PPC64: return addThunkPPC64(ctx, rel.type, s, a); + case EM_HEXAGON: + return addThunkHexagon(ctx, isec, rel, s); default: - llvm_unreachable("add Thunk only supported for ARM, AVR, Mips and PowerPC"); + llvm_unreachable( + "add Thunk only supported for ARM, AVR, Hexagon, Mips and PowerPC"); } } diff --git a/lld/docs/DTLTO.rst b/lld/docs/DTLTO.rst index 985decf6c7db..54fcc034d137 100644 --- a/lld/docs/DTLTO.rst +++ b/lld/docs/DTLTO.rst @@ -7,8 +7,7 @@ during the traditional link step. The implementation is documented here: https://llvm.org/docs/DTLTO.html. -Currently, DTLTO is only supported in ELF LLD. Support will be added to other -LLD flavours in the future. +Currently, DTLTO is only supported in ELF and COFF LLD. ELF LLD ------- @@ -40,3 +39,37 @@ The command-line interface is as follows: Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported. Currently, other options are silently accepted but do not have the intended effect. Support for such options will be expanded in the future. + +COFF LLD +-------- + +The command-line interface is as follows: + +- ``/thinlto-distributor:<path>`` + Specifies the file to execute as the distributor process. If specified, + ThinLTO backend compilations will be distributed. + +- ``/thinlto-remote-compiler:<path>`` + Specifies the path to the compiler that the distributor process will use for + backend compilations. The compiler invoked must match the version of LLD. + +- ``/thinlto-distributor-arg:<arg>`` + Specifies ``<arg>`` on the command line when invoking the distributor. + Can be specified multiple times. + +- ``/thinlto-remote-compiler-arg:<arg>`` + Appends ``<arg>`` to the remote compiler's command line. + Can be specified multiple times. + + Options that introduce extra input/output files may cause miscompilation if + the distribution system does not automatically handle pushing/fetching them to + remote nodes. In such cases, configure the distributor - possibly using + ``/thinlto-distributor-arg:`` - to manage these dependencies. See the + distributor documentation for details. + +Some LLD LTO options (e.g., ``/lto-sample-profile:<file>``) are supported. +Currently, other options are silently accepted but do not have the intended +effect. Support for such options could be expanded in the future. + +Currently, there is no DTLTO command line interface supplied for ``clang-cl``, +as users are expected to invoke LLD directly.
\ No newline at end of file diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst index 863b20189100..03671c6f3b6f 100644 --- a/lld/docs/ReleaseNotes.rst +++ b/lld/docs/ReleaseNotes.rst @@ -92,6 +92,9 @@ Breaking changes COFF Improvements ----------------- +* ``/thinlto-distributor`` and ``/thinlto-remote-compiler`` options + added to support Integrated Distributed ThinLTO. + (`#147265 <https://github.com/llvm/llvm-project/pull/147265>`_) MinGW Improvements ------------------ diff --git a/lldb/include/lldb/Target/InstrumentationRuntimeStopInfo.h b/lldb/include/lldb/Target/InstrumentationRuntimeStopInfo.h index 534516085091..dafa41c11327 100644 --- a/lldb/include/lldb/Target/InstrumentationRuntimeStopInfo.h +++ b/lldb/include/lldb/Target/InstrumentationRuntimeStopInfo.h @@ -24,6 +24,9 @@ public: return lldb::eStopReasonInstrumentation; } + std::optional<uint32_t> + GetSuggestedStackFrameIndex(bool inlined_stack) override; + const char *GetDescription() override; bool DoShouldNotify(Event *event_ptr) override { return true; } diff --git a/lldb/include/lldb/Target/StackFrameList.h b/lldb/include/lldb/Target/StackFrameList.h index 8a66296346f2..8d455dc831df 100644 --- a/lldb/include/lldb/Target/StackFrameList.h +++ b/lldb/include/lldb/Target/StackFrameList.h @@ -46,6 +46,9 @@ public: /// Mark a stack frame as the currently selected frame and return its index. uint32_t SetSelectedFrame(lldb_private::StackFrame *frame); + /// Resets the selected frame index of this object. + void ClearSelectedFrameIndex(); + /// Get the currently selected frame index. /// We should only call SelectMostRelevantFrame if (a) the user hasn't already /// selected a frame, and (b) if this really is a user facing @@ -172,6 +175,15 @@ protected: /// The currently selected frame. An optional is used to record whether anyone /// has set the selected frame on this stack yet. We only let recognizers /// change the frame if this is the first time GetSelectedFrame is called. + /// + /// Thread-safety: + /// This member is not protected by a mutex. + /// LLDB really only should have an opinion about the selected frame index + /// when a process stops, before control gets handed back to the user. + /// After that, it's up to them to change it whenever they feel like it. + /// If two parts of lldb decided they wanted to be in control of the selected + /// frame index on stop the right way to fix it would need to be some explicit + /// negotiation for who gets to control this. std::optional<uint32_t> m_selected_frame_idx; /// The number of concrete frames fetched while filling the frame list. This diff --git a/lldb/include/lldb/Target/Thread.h b/lldb/include/lldb/Target/Thread.h index 6ede7fa301a8..688c056da263 100644 --- a/lldb/include/lldb/Target/Thread.h +++ b/lldb/include/lldb/Target/Thread.h @@ -479,6 +479,11 @@ public: bool SetSelectedFrameByIndexNoisily(uint32_t frame_idx, Stream &output_stream); + /// Resets the selected frame index of this object. + void ClearSelectedFrameIndex() { + return GetStackFrameList()->ClearSelectedFrameIndex(); + } + void SetDefaultFileAndLineToSelectedFrame() { GetStackFrameList()->SetDefaultFileAndLineToSelectedFrame(); } diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp index 2529e78f78bc..633b860b5784 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp @@ -1053,6 +1053,16 @@ ClangASTImporter::MapCompleter::~MapCompleter() = default; llvm::Expected<Decl *> ClangASTImporter::ASTImporterDelegate::ImportImpl(Decl *From) { + // FIXME: The Minimal import mode of clang::ASTImporter does not correctly + // import Lambda definitions. Work around this for now by not importing + // lambdas at all. This is most likely encountered when importing decls from + // the `std` module (not from debug-info), where lambdas can be defined in + // inline function bodies. Those will be imported by LLDB. + if (const auto *CXX = llvm::dyn_cast<clang::CXXRecordDecl>(From)) + if (CXX->isLambda()) + return llvm::make_error<ASTImportError>( + ASTImportError::UnsupportedConstruct); + if (m_std_handler) { std::optional<Decl *> D = m_std_handler->Import(From); if (D) { diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp index 9a869f3ea028..862082f8a16b 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp @@ -790,31 +790,27 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) { lldb_private::formatters::LibcxxStringSummaryProviderASCII, "std::string summary provider", "^std::__[[:alnum:]]+::basic_string<char, " - "std::__[[:alnum:]]+::char_traits<char>, " - "std::__[[:alnum:]]+::allocator<char> >$", + "std::__[[:alnum:]]+::char_traits<char>,.*>$", stl_summary_flags, true); AddCXXSummary(cpp_category_sp, lldb_private::formatters::LibcxxStringSummaryProviderASCII, "std::string summary provider", "^std::__[[:alnum:]]+::basic_string<unsigned char, " - "std::__[[:alnum:]]+::char_traits<unsigned char>, " - "std::__[[:alnum:]]+::allocator<unsigned char> >$", + "std::__[[:alnum:]]+::char_traits<unsigned char>,.*>$", stl_summary_flags, true); AddCXXSummary(cpp_category_sp, lldb_private::formatters::LibcxxStringSummaryProviderUTF16, "std::u16string summary provider", "^std::__[[:alnum:]]+::basic_string<char16_t, " - "std::__[[:alnum:]]+::char_traits<char16_t>, " - "std::__[[:alnum:]]+::allocator<char16_t> >$", + "std::__[[:alnum:]]+::char_traits<char16_t>,.*>$", stl_summary_flags, true); AddCXXSummary(cpp_category_sp, lldb_private::formatters::LibcxxStringSummaryProviderUTF32, "std::u32string summary provider", "^std::__[[:alnum:]]+::basic_string<char32_t, " - "std::__[[:alnum:]]+::char_traits<char32_t>, " - "std::__[[:alnum:]]+::allocator<char32_t> >$", + "std::__[[:alnum:]]+::char_traits<char32_t>,.*>$", stl_summary_flags, true); AddCXXSummary(cpp_category_sp, @@ -825,8 +821,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) { lldb_private::formatters::LibcxxWStringSummaryProvider, "std::wstring summary provider", "^std::__[[:alnum:]]+::basic_string<wchar_t, " - "std::__[[:alnum:]]+::char_traits<wchar_t>, " - "std::__[[:alnum:]]+::allocator<wchar_t> >$", + "std::__[[:alnum:]]+::char_traits<wchar_t>,.*>$", stl_summary_flags, true); AddCXXSummary(cpp_category_sp, @@ -1342,24 +1337,16 @@ static void RegisterStdStringSummaryProvider( category_sp->AddTypeSummary(makeSpecifier(string_ty), summary_sp); - // std::basic_string<char> category_sp->AddTypeSummary( makeSpecifier(llvm::formatv("std::basic_string<{}>", char_ty).str()), summary_sp); - // std::basic_string<char,std::char_traits<char>,std::allocator<char> > - category_sp->AddTypeSummary( - makeSpecifier(llvm::formatv("std::basic_string<{0},std::char_traits<{0}>," - "std::allocator<{0}> >", - char_ty) - .str()), - summary_sp); - // std::basic_string<char, std::char_traits<char>, std::allocator<char> > + category_sp->AddTypeSummary( - makeSpecifier( - llvm::formatv("std::basic_string<{0}, std::char_traits<{0}>, " - "std::allocator<{0}> >", + std::make_shared<lldb_private::TypeNameSpecifierImpl>( + llvm::formatv("^std::basic_string<{0}, ?std::char_traits<{0}>,.*>$", char_ty) - .str()), + .str(), + eFormatterMatchRegex), summary_sp); } @@ -1382,20 +1369,17 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) { cpp_category_sp->AddTypeSummary("std::__cxx11::string", eFormatterMatchExact, string_summary_sp); cpp_category_sp->AddTypeSummary( - "std::__cxx11::basic_string<char, std::char_traits<char>, " - "std::allocator<char> >", - eFormatterMatchExact, string_summary_sp); - cpp_category_sp->AddTypeSummary("std::__cxx11::basic_string<unsigned char, " - "std::char_traits<unsigned char>, " - "std::allocator<unsigned char> >", - eFormatterMatchExact, string_summary_sp); + "^std::__cxx11::basic_string<char, std::char_traits<char>,.*>$", + eFormatterMatchRegex, string_summary_sp); + cpp_category_sp->AddTypeSummary("^std::__cxx11::basic_string<unsigned char, " + "std::char_traits<unsigned char>,.*>$", + eFormatterMatchRegex, string_summary_sp); cpp_category_sp->AddTypeSummary("std::__cxx11::wstring", eFormatterMatchExact, string_summary_sp); cpp_category_sp->AddTypeSummary( - "std::__cxx11::basic_string<wchar_t, std::char_traits<wchar_t>, " - "std::allocator<wchar_t> >", - eFormatterMatchExact, string_summary_sp); + "^std::__cxx11::basic_string<wchar_t, std::char_traits<wchar_t>,.*>$", + eFormatterMatchRegex, string_summary_sp); SyntheticChildren::Flags stl_synth_flags; stl_synth_flags.SetCascades(true).SetSkipPointers(false).SetSkipReferences( diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp index a7874047942c..6053d042b29b 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp @@ -49,11 +49,6 @@ static void consumeInlineNamespace(llvm::StringRef &name) { } } -bool lldb_private::formatters::isOldCompressedPairLayout( - ValueObject &pair_obj) { - return isStdTemplate(pair_obj.GetTypeName(), "__compressed_pair"); -} - bool lldb_private::formatters::isStdTemplate(ConstString type_name, llvm::StringRef type) { llvm::StringRef name = type_name.GetStringRef(); @@ -105,6 +100,44 @@ lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair( return value; } +std::pair<lldb::ValueObjectSP, bool> +lldb_private::formatters::GetValueOrOldCompressedPair( + ValueObject &obj, size_t anon_struct_idx, llvm::StringRef child_name, + llvm::StringRef compressed_pair_name) { + auto is_old_compressed_pair = [](ValueObject &pair_obj) -> bool { + return isStdTemplate(pair_obj.GetTypeName(), "__compressed_pair"); + }; + + // Try searching the child member in an anonymous structure first. + if (auto unwrapped = obj.GetChildAtIndex(anon_struct_idx)) { + ValueObjectSP node_sp(obj.GetChildMemberWithName(child_name)); + if (node_sp) + return {node_sp, is_old_compressed_pair(*node_sp)}; + } + + // Older versions of libc++ don't wrap the children in anonymous structures. + // Try that instead. + ValueObjectSP node_sp(obj.GetChildMemberWithName(child_name)); + if (node_sp) + return {node_sp, is_old_compressed_pair(*node_sp)}; + + // Try the even older __compressed_pair layout. + + assert(!compressed_pair_name.empty()); + + node_sp = obj.GetChildMemberWithName(compressed_pair_name); + + // Unrecognized layout (possibly older than LLDB supports). + if (!node_sp) + return {nullptr, false}; + + // Expected old compressed_pair layout, but got something else. + if (!is_old_compressed_pair(*node_sp)) + return {nullptr, false}; + + return {node_sp, true}; +} + bool lldb_private::formatters::LibcxxFunctionSummaryProvider( ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) { @@ -205,11 +238,12 @@ bool lldb_private::formatters::LibcxxUniquePointerSummaryProvider( if (!valobj_sp) return false; - ValueObjectSP ptr_sp(valobj_sp->GetChildMemberWithName("__ptr_")); + auto [ptr_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + *valobj_sp, /*anon_struct_idx=*/0, "__ptr_", "__ptr_"); if (!ptr_sp) return false; - if (isOldCompressedPairLayout(*ptr_sp)) + if (is_compressed_pair) ptr_sp = GetFirstValueOfLibCXXCompressedPair(*ptr_sp); if (!ptr_sp) @@ -379,13 +413,14 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::Update() { if (!valobj_sp) return lldb::ChildCacheState::eRefetch; - ValueObjectSP ptr_sp(valobj_sp->GetChildMemberWithName("__ptr_")); + auto [ptr_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + *valobj_sp, /*anon_struct_idx=*/0, "__ptr_", "__ptr_"); if (!ptr_sp) return lldb::ChildCacheState::eRefetch; // Retrieve the actual pointer and the deleter, and clone them to give them // user-friendly names. - if (isOldCompressedPairLayout(*ptr_sp)) { + if (is_compressed_pair) { if (ValueObjectSP value_pointer_sp = GetFirstValueOfLibCXXCompressedPair(*ptr_sp)) m_value_ptr_sp = value_pointer_sp->Clone(ConstString("pointer")); @@ -424,17 +459,15 @@ enum class StringLayout { CSD, DSC }; } static ValueObjectSP ExtractLibCxxStringData(ValueObject &valobj) { - if (auto rep_sp = valobj.GetChildMemberWithName("__rep_")) - return rep_sp; - - ValueObjectSP valobj_r_sp = valobj.GetChildMemberWithName("__r_"); - if (!valobj_r_sp || !valobj_r_sp->GetError().Success()) + auto [valobj_r_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + valobj, /*anon_struct_idx=*/0, "__rep_", "__r_"); + if (!valobj_r_sp) return nullptr; - if (!isOldCompressedPairLayout(*valobj_r_sp)) - return nullptr; + if (is_compressed_pair) + return GetFirstValueOfLibCXXCompressedPair(*valobj_r_sp); - return GetFirstValueOfLibCXXCompressedPair(*valobj_r_sp); + return valobj_r_sp; } /// Determine the size in bytes of \p valobj (a libc++ std::string object) and diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h index d88a6ecb1fa8..819f8a985f9b 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h @@ -25,7 +25,22 @@ GetChildMemberWithName(ValueObject &obj, lldb::ValueObjectSP GetFirstValueOfLibCXXCompressedPair(ValueObject &pair); lldb::ValueObjectSP GetSecondValueOfLibCXXCompressedPair(ValueObject &pair); -bool isOldCompressedPairLayout(ValueObject &pair_obj); + +/// Returns the ValueObjectSP of the child of \c obj. If \c obj has no +/// child named \c child_name, returns the __compressed_pair child instead +/// with \c compressed_pair_name, if one exists. +/// +/// Latest libc++ wrap the compressed children in an anonymous structure. +/// The \c anon_struct_idx indicates the location of this struct. +/// +/// The returned boolean is \c true if the returned child was has an old-style +/// libc++ __compressed_pair layout. +/// +/// If no child was found returns a nullptr. +std::pair<lldb::ValueObjectSP, bool> +GetValueOrOldCompressedPair(ValueObject &obj, size_t anon_struct_idx, + llvm::StringRef child_name, + llvm::StringRef compressed_pair_name); bool isStdTemplate(ConstString type_name, llvm::StringRef type); bool LibcxxStringSummaryProviderASCII( diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp index 826e6ab090e1..dfc23266fc14 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp @@ -297,11 +297,18 @@ lldb::ChildCacheState ForwardListFrontEnd::Update() { if (err.Fail() || !backend_addr) return lldb::ChildCacheState::eRefetch; - ValueObjectSP impl_sp(m_backend.GetChildMemberWithName("__before_begin_")); + auto list_base_sp = m_backend.GetChildAtIndex(0); + if (!list_base_sp) + return lldb::ChildCacheState::eRefetch; + + // Anonymous strucutre index is in base class at index 0. + auto [impl_sp, is_compressed_pair] = + GetValueOrOldCompressedPair(*list_base_sp, /*anon_struct_idx=*/0, + "__before_begin_", "__before_begin_"); if (!impl_sp) return ChildCacheState::eRefetch; - if (isOldCompressedPairLayout(*impl_sp)) + if (is_compressed_pair) impl_sp = GetFirstValueOfLibCXXCompressedPair(*impl_sp); if (!impl_sp) @@ -324,17 +331,10 @@ llvm::Expected<uint32_t> ListFrontEnd::CalculateNumChildren() { if (!m_head || !m_tail || m_node_address == 0) return 0; - ValueObjectSP size_node_sp(m_backend.GetChildMemberWithName("__size_")); - if (!size_node_sp) { - size_node_sp = m_backend.GetChildMemberWithName( - "__size_alloc_"); // pre-compressed_pair rework - - if (!isOldCompressedPairLayout(*size_node_sp)) - return llvm::createStringError("Unexpected std::list layout: expected " - "old __compressed_pair layout."); - + auto [size_node_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + m_backend, /*anon_struct_idx=*/1, "__size_", "__size_alloc_"); + if (is_compressed_pair) size_node_sp = GetFirstValueOfLibCXXCompressedPair(*size_node_sp); - } if (size_node_sp) m_count = size_node_sp->GetValueAsUnsigned(UINT32_MAX); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp index 41441dfbc718..85766966f155 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp @@ -200,7 +200,8 @@ public: llvm::Expected<size_t> GetIndexOfChildWithName(ConstString name) override; private: - llvm::Expected<uint32_t> CalculateNumChildrenForOldCompressedPairLayout(); + llvm::Expected<uint32_t> + CalculateNumChildrenForOldCompressedPairLayout(ValueObject &pair); /// Returns the ValueObject for the __tree_node type that /// holds the key/value pair of the node at index \ref idx. @@ -254,16 +255,8 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd:: llvm::Expected<uint32_t> lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd:: - CalculateNumChildrenForOldCompressedPairLayout() { - ValueObjectSP node_sp(m_tree->GetChildMemberWithName("__pair3_")); - if (!node_sp) - return 0; - - if (!isOldCompressedPairLayout(*node_sp)) - return llvm::createStringError("Unexpected std::map layout: expected " - "old __compressed_pair layout."); - - node_sp = GetFirstValueOfLibCXXCompressedPair(*node_sp); + CalculateNumChildrenForOldCompressedPairLayout(ValueObject &pair) { + auto node_sp = GetFirstValueOfLibCXXCompressedPair(pair); if (!node_sp) return 0; @@ -281,12 +274,16 @@ llvm::Expected<uint32_t> lldb_private::formatters:: if (m_tree == nullptr) return 0; - if (auto node_sp = m_tree->GetChildMemberWithName("__size_")) { - m_count = node_sp->GetValueAsUnsigned(0); - return m_count; - } + auto [size_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + *m_tree, /*anon_struct_idx=*/2, "__size_", "__pair3_"); + if (!size_sp) + return llvm::createStringError("Unexpected std::map layout"); - return CalculateNumChildrenForOldCompressedPairLayout(); + if (is_compressed_pair) + return CalculateNumChildrenForOldCompressedPairLayout(*size_sp); + + m_count = size_sp->GetValueAsUnsigned(0); + return m_count; } ValueObjectSP diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp index 501fd0945b82..f88a5319068a 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp @@ -130,22 +130,17 @@ CompilerType lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd:: CompilerType lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd:: GetNodeType() { - auto node_sp = m_backend.GetChildAtNamePath({"__table_", "__first_node_"}); - - if (!node_sp) { - auto p1_sp = m_backend.GetChildAtNamePath({"__table_", "__p1_"}); - if (!p1_sp) - return {}; + auto table_sp = m_backend.GetChildMemberWithName("__table_"); + if (!table_sp) + return {}; - if (!isOldCompressedPairLayout(*p1_sp)) - return {}; + auto [node_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + *table_sp, /*anon_struct_idx=*/1, "__first_node_", "__p1_"); + if (is_compressed_pair) + node_sp = GetFirstValueOfLibCXXCompressedPair(*node_sp); - node_sp = GetFirstValueOfLibCXXCompressedPair(*p1_sp); - if (!node_sp) - return {}; - } - - assert(node_sp); + if (!node_sp) + return {}; return node_sp->GetCompilerType().GetTypeTemplateArgument(0).GetPointeeType(); } @@ -223,19 +218,15 @@ lldb::ValueObjectSP lldb_private::formatters:: llvm::Expected<size_t> lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd:: CalculateNumChildrenImpl(ValueObject &table) { - if (auto size_sp = table.GetChildMemberWithName("__size_")) + auto [size_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + table, /*anon_struct_idx=*/2, "__size_", "__p2_"); + if (!is_compressed_pair && size_sp) return size_sp->GetValueAsUnsigned(0); - ValueObjectSP p2_sp = table.GetChildMemberWithName("__p2_"); - if (!p2_sp) - return llvm::createStringError( - "Unexpected std::unordered_map layout: __p2_ member not found."); + if (!is_compressed_pair) + return llvm::createStringError("Unsupported std::unordered_map layout."); - if (!isOldCompressedPairLayout(*p2_sp)) - return llvm::createStringError("Unexpected std::unordered_map layout: old " - "__compressed_pair layout not found."); - - ValueObjectSP num_elements_sp = GetFirstValueOfLibCXXCompressedPair(*p2_sp); + ValueObjectSP num_elements_sp = GetFirstValueOfLibCXXCompressedPair(*size_sp); if (!num_elements_sp) return llvm::createStringError( @@ -246,19 +237,13 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd:: } static ValueObjectSP GetTreePointer(ValueObject &table) { - ValueObjectSP tree_sp = table.GetChildMemberWithName("__first_node_"); - if (!tree_sp) { - ValueObjectSP p1_sp = table.GetChildMemberWithName("__p1_"); - if (!p1_sp) - return nullptr; - - if (!isOldCompressedPairLayout(*p1_sp)) - return nullptr; - - tree_sp = GetFirstValueOfLibCXXCompressedPair(*p1_sp); - if (!tree_sp) - return nullptr; - } + auto [tree_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + table, /*anon_struct_idx=*/1, "__first_node_", "__p1_"); + if (is_compressed_pair) + tree_sp = GetFirstValueOfLibCXXCompressedPair(*tree_sp); + + if (!tree_sp) + return nullptr; return tree_sp->GetChildMemberWithName("__next_"); } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp index 4bcdf01c221a..60913e5c1ac5 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp @@ -126,17 +126,15 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::GetChildAtIndex( } static ValueObjectSP GetDataPointer(ValueObject &root) { - if (auto cap_sp = root.GetChildMemberWithName("__cap_")) - return cap_sp; - - ValueObjectSP cap_sp = root.GetChildMemberWithName("__end_cap_"); + auto [cap_sp, is_compressed_pair] = GetValueOrOldCompressedPair( + root, /*anon_struct_idx=*/2, "__cap_", "__end_cap_"); if (!cap_sp) return nullptr; - if (!isOldCompressedPairLayout(*cap_sp)) - return nullptr; + if (is_compressed_pair) + return GetFirstValueOfLibCXXCompressedPair(*cap_sp); - return GetFirstValueOfLibCXXCompressedPair(*cap_sp); + return cap_sp; } lldb::ChildCacheState diff --git a/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp b/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp index 7adc00622ec2..d21dac221aa2 100644 --- a/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp +++ b/lldb/source/Plugins/Process/FreeBSD/NativeRegisterContextFreeBSD_arm64.cpp @@ -44,7 +44,8 @@ NativeRegisterContextFreeBSD::CreateHostNativeRegisterContextFreeBSD( NativeProcessFreeBSD &process = native_thread.GetProcess(); g_register_flags_detector.DetectFields( process.GetAuxValue(AuxVector::AUXV_FREEBSD_AT_HWCAP).value_or(0), - process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2).value_or(0)); + process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2).value_or(0), + /*hwcap3=*/0); } return new NativeRegisterContextFreeBSD_arm64(target_arch, native_thread); diff --git a/lldb/source/Plugins/Process/Utility/AuxVector.cpp b/lldb/source/Plugins/Process/Utility/AuxVector.cpp index f495ffb1924e..50500a8593e1 100644 --- a/lldb/source/Plugins/Process/Utility/AuxVector.cpp +++ b/lldb/source/Plugins/Process/Utility/AuxVector.cpp @@ -84,6 +84,7 @@ const char *AuxVector::GetEntryName(EntryType type) const { case ENTRY_NAME(AUXV_AT_BASE_PLATFORM); break; case ENTRY_NAME(AUXV_AT_RANDOM); break; case ENTRY_NAME(AUXV_AT_HWCAP2); break; + case ENTRY_NAME(AUXV_AT_HWCAP3); break; case ENTRY_NAME(AUXV_AT_EXECFN); break; case ENTRY_NAME(AUXV_AT_SYSINFO); break; case ENTRY_NAME(AUXV_AT_SYSINFO_EHDR); break; diff --git a/lldb/source/Plugins/Process/Utility/AuxVector.h b/lldb/source/Plugins/Process/Utility/AuxVector.h index 2670b34f6b0a..7733e0ffc683 100644 --- a/lldb/source/Plugins/Process/Utility/AuxVector.h +++ b/lldb/source/Plugins/Process/Utility/AuxVector.h @@ -57,6 +57,7 @@ public: AUXV_AT_BASE_PLATFORM = 24, ///< String identifying real platforms. AUXV_AT_RANDOM = 25, ///< Address of 16 random bytes. AUXV_AT_HWCAP2 = 26, ///< Extension of AT_HWCAP. + AUXV_AT_HWCAP3 = 29, ///< Extension of AT_HWCAP. AUXV_AT_EXECFN = 31, ///< Filename of executable. AUXV_AT_SYSINFO = 32, ///< Pointer to the global system page used for system /// calls and other nice things. diff --git a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp index 042940b7dff6..330a24af67c4 100644 --- a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp +++ b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp @@ -26,11 +26,15 @@ #define HWCAP2_EBF16 (1ULL << 32) #define HWCAP2_FPMR (1ULL << 48) +#define HWCAP3_MTE_STORE_ONLY (1ULL << 1) + using namespace lldb_private; Arm64RegisterFlagsDetector::Fields -Arm64RegisterFlagsDetector::DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2) { +Arm64RegisterFlagsDetector::DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3) { (void)hwcap; + (void)hwcap3; if (!(hwcap2 & HWCAP2_FPMR)) return {}; @@ -53,8 +57,10 @@ Arm64RegisterFlagsDetector::DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2) { Arm64RegisterFlagsDetector::Fields Arm64RegisterFlagsDetector::DetectGCSFeatureFields(uint64_t hwcap, - uint64_t hwcap2) { + uint64_t hwcap2, + uint64_t hwcap3) { (void)hwcap2; + (void)hwcap3; if (!(hwcap & HWCAP_GCS)) return {}; @@ -67,8 +73,10 @@ Arm64RegisterFlagsDetector::DetectGCSFeatureFields(uint64_t hwcap, } Arm64RegisterFlagsDetector::Fields -Arm64RegisterFlagsDetector::DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2) { +Arm64RegisterFlagsDetector::DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3) { (void)hwcap; + (void)hwcap3; if (!(hwcap2 & HWCAP2_SME)) return {}; @@ -83,8 +91,8 @@ Arm64RegisterFlagsDetector::DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2) { } Arm64RegisterFlagsDetector::Fields -Arm64RegisterFlagsDetector::DetectMTECtrlFields(uint64_t hwcap, - uint64_t hwcap2) { +Arm64RegisterFlagsDetector::DetectMTECtrlFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3) { (void)hwcap; if (!(hwcap2 & HWCAP2_MTE)) @@ -94,16 +102,29 @@ Arm64RegisterFlagsDetector::DetectMTECtrlFields(uint64_t hwcap, // to prctl(PR_TAGGED_ADDR_CTRL...). Fields are derived from the defines // used to build the value. + std::vector<RegisterFlags::Field> fields; + fields.reserve(4); + if (hwcap3 & HWCAP3_MTE_STORE_ONLY) + fields.push_back({"STORE_ONLY", 19}); + static const FieldEnum tcf_enum( "tcf_enum", {{0, "TCF_NONE"}, {1, "TCF_SYNC"}, {2, "TCF_ASYNC"}, {3, "TCF_ASYMM"}}); - return {{"TAGS", 3, 18}, // 16 bit bitfield shifted up by PR_MTE_TAG_SHIFT. - {"TCF", 1, 2, &tcf_enum}, - {"TAGGED_ADDR_ENABLE", 0}}; + + fields.insert( + std::end(fields), + {{"TAGS", 3, 18}, // 16 bit bitfield shifted up by PR_MTE_TAG_SHIFT. + {"TCF", 1, 2, &tcf_enum}, + {"TAGGED_ADDR_ENABLE", 0}}); + + return fields; } Arm64RegisterFlagsDetector::Fields -Arm64RegisterFlagsDetector::DetectFPCRFields(uint64_t hwcap, uint64_t hwcap2) { +Arm64RegisterFlagsDetector::DetectFPCRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3) { + (void)hwcap3; + static const FieldEnum rmode_enum( "rmode_enum", {{0, "RN"}, {1, "RP"}, {2, "RM"}, {3, "RZ"}}); @@ -142,10 +163,12 @@ Arm64RegisterFlagsDetector::DetectFPCRFields(uint64_t hwcap, uint64_t hwcap2) { } Arm64RegisterFlagsDetector::Fields -Arm64RegisterFlagsDetector::DetectFPSRFields(uint64_t hwcap, uint64_t hwcap2) { +Arm64RegisterFlagsDetector::DetectFPSRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3) { // fpsr's contents are constant. (void)hwcap; (void)hwcap2; + (void)hwcap3; return { // Bits 31-28 are N/Z/C/V, only used by AArch32. @@ -162,7 +185,10 @@ Arm64RegisterFlagsDetector::DetectFPSRFields(uint64_t hwcap, uint64_t hwcap2) { } Arm64RegisterFlagsDetector::Fields -Arm64RegisterFlagsDetector::DetectCPSRFields(uint64_t hwcap, uint64_t hwcap2) { +Arm64RegisterFlagsDetector::DetectCPSRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3) { + (void)hwcap3; + // The fields here are a combination of the Arm manual's SPSR_EL1, // plus a few changes where Linux has decided not to make use of them at all, // or at least not from userspace. @@ -207,9 +233,10 @@ Arm64RegisterFlagsDetector::DetectCPSRFields(uint64_t hwcap, uint64_t hwcap2) { return cpsr_fields; } -void Arm64RegisterFlagsDetector::DetectFields(uint64_t hwcap, uint64_t hwcap2) { +void Arm64RegisterFlagsDetector::DetectFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3) { for (auto ® : m_registers) - reg.m_flags.SetFields(reg.m_detector(hwcap, hwcap2)); + reg.m_flags.SetFields(reg.m_detector(hwcap, hwcap2, hwcap3)); m_has_detected = true; } diff --git a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h index 7daebcc71db0..aec2bf9f4886 100644 --- a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h +++ b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h @@ -40,7 +40,7 @@ public: /// If called more than once, fields will be redetected each time from /// scratch. If the target would not have this register at all, the list of /// fields will be left empty. - void DetectFields(uint64_t hwcap, uint64_t hwcap2); + void DetectFields(uint64_t hwcap, uint64_t hwcap2, uint64_t hwcap3); /// Add the field information of any registers named in this class, /// to the relevant RegisterInfo instances. Note that this will be done @@ -53,15 +53,22 @@ public: private: using Fields = std::vector<RegisterFlags::Field>; - using DetectorFn = std::function<Fields(uint64_t, uint64_t)>; + using DetectorFn = std::function<Fields(uint64_t, uint64_t, uint64_t)>; - static Fields DetectCPSRFields(uint64_t hwcap, uint64_t hwcap2); - static Fields DetectFPSRFields(uint64_t hwcap, uint64_t hwcap2); - static Fields DetectFPCRFields(uint64_t hwcap, uint64_t hwcap2); - static Fields DetectMTECtrlFields(uint64_t hwcap, uint64_t hwcap2); - static Fields DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2); - static Fields DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2); - static Fields DetectGCSFeatureFields(uint64_t hwcap, uint64_t hwcap2); + static Fields DetectCPSRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3); + static Fields DetectFPSRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3); + static Fields DetectFPCRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3); + static Fields DetectMTECtrlFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3); + static Fields DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3); + static Fields DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3); + static Fields DetectGCSFeatureFields(uint64_t hwcap, uint64_t hwcap2, + uint64_t hwcap3); struct RegisterEntry { RegisterEntry(llvm::StringRef name, unsigned size, DetectorFn detector) diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp index bd02bb0e69a4..d5046d369ab2 100644 --- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp +++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp @@ -96,14 +96,19 @@ RegisterContextCorePOSIX_arm64::RegisterContextCorePOSIX_arm64( llvm::Triple::OSType os = process->GetArchitecture().GetTriple().getOS(); if ((os == llvm::Triple::Linux) || (os == llvm::Triple::FreeBSD)) { AuxVector aux_vec(process->GetAuxvData()); - std::optional<uint64_t> auxv_at_hwcap = aux_vec.GetAuxValue( - os == llvm::Triple::FreeBSD ? AuxVector::AUXV_FREEBSD_AT_HWCAP - : AuxVector::AUXV_AT_HWCAP); + bool is_freebsd = os == llvm::Triple::FreeBSD; + std::optional<uint64_t> auxv_at_hwcap = + aux_vec.GetAuxValue(is_freebsd ? AuxVector::AUXV_FREEBSD_AT_HWCAP + : AuxVector::AUXV_AT_HWCAP); std::optional<uint64_t> auxv_at_hwcap2 = aux_vec.GetAuxValue(AuxVector::AUXV_AT_HWCAP2); + std::optional<uint64_t> auxv_at_hwcap3 = + is_freebsd ? std::nullopt + : aux_vec.GetAuxValue(AuxVector::AUXV_AT_HWCAP3); m_register_flags_detector.DetectFields(auxv_at_hwcap.value_or(0), - auxv_at_hwcap2.value_or(0)); + auxv_at_hwcap2.value_or(0), + auxv_at_hwcap3.value_or(0)); m_register_flags_detector.UpdateRegisterInfo(GetRegisterInfo(), GetRegisterCount()); } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp index c76d67b47b33..8916c58beec0 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp @@ -2201,6 +2201,18 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die, for (DelayedAddObjCClassProperty &property : delayed_properties) property.Finalize(); } + } else if (Language::LanguageIsObjC( + static_cast<LanguageType>(die.GetAttributeValueAsUnsigned( + DW_AT_APPLE_runtime_class, eLanguageTypeUnknown)))) { + /// The forward declaration was C++ but the definition is Objective-C. + /// We currently don't handle such situations. In such cases, keep the + /// forward declaration without a definition to avoid violating Clang AST + /// invariants. + LLDB_LOG(GetLog(LLDBLog::Expressions), + "WARNING: Type completion aborted because forward declaration for " + "'{0}' is C++ while definition is Objective-C.", + llvm::StringRef(die.GetName())); + return {}; } if (!bases.empty()) { diff --git a/lldb/source/Target/InstrumentationRuntimeStopInfo.cpp b/lldb/source/Target/InstrumentationRuntimeStopInfo.cpp index 7f82581cc601..aef895def793 100644 --- a/lldb/source/Target/InstrumentationRuntimeStopInfo.cpp +++ b/lldb/source/Target/InstrumentationRuntimeStopInfo.cpp @@ -8,13 +8,20 @@ #include "lldb/Target/InstrumentationRuntimeStopInfo.h" +#include "lldb/Core/Module.h" #include "lldb/Target/InstrumentationRuntime.h" #include "lldb/Target/Process.h" +#include "lldb/lldb-enumerations.h" #include "lldb/lldb-private.h" using namespace lldb; using namespace lldb_private; +static bool IsStoppedInDarwinSanitizer(Thread &thread, Module &module) { + return module.GetFileSpec().GetFilename().GetStringRef().starts_with( + "libclang_rt."); +} + InstrumentationRuntimeStopInfo::InstrumentationRuntimeStopInfo( Thread &thread, std::string description, StructuredData::ObjectSP additional_data) @@ -34,3 +41,38 @@ InstrumentationRuntimeStopInfo::CreateStopReasonWithInstrumentationData( return StopInfoSP( new InstrumentationRuntimeStopInfo(thread, description, additionalData)); } + +std::optional<uint32_t> +InstrumentationRuntimeStopInfo::GetSuggestedStackFrameIndex( + bool inlined_stack) { + ThreadSP thread_sp = GetThread(); + if (!thread_sp) + return std::nullopt; + + // Defensive upper-bound of when we stop walking up the frames in + // case we somehow ended up looking at an infinite recursion. + constexpr size_t max_stack_depth = 128; + + // Start at parent frame. + size_t stack_idx = 1; + StackFrameSP most_relevant_frame_sp = + thread_sp->GetStackFrameAtIndex(stack_idx); + + while (most_relevant_frame_sp && stack_idx <= max_stack_depth) { + auto const &sc = + most_relevant_frame_sp->GetSymbolContext(lldb::eSymbolContextModule); + + if (!sc.module_sp) + return std::nullopt; + + // Found a frame outside of the sanitizer runtime libraries. + // That's the one we want to display. + if (!IsStoppedInDarwinSanitizer(*thread_sp, *sc.module_sp)) + return stack_idx; + + ++stack_idx; + most_relevant_frame_sp = thread_sp->GetStackFrameAtIndex(stack_idx); + } + + return stack_idx; +} diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp index 2aa02fd58335..a2fa88b56913 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -4257,6 +4257,14 @@ bool Process::ProcessEventData::ShouldStop(Event *event_ptr, // appropriately. We also need to stop processing actions, since they // aren't expecting the target to be running. + // Clear the selected frame which may have been set as part of utility + // expressions that have been run as part of this stop. If we didn't + // clear this, then StopInfo::GetSuggestedStackFrameIndex would not + // take affect when we next called SelectMostRelevantFrame. + // PerformAction should not be the one setting a selected frame, instead + // this should be done via GetSuggestedStackFrameIndex. + thread_sp->ClearSelectedFrameIndex(); + // FIXME: we might have run. if (stop_info_sp->HasTargetRunSinceMe()) { SetRestarted(true); diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp index 16cd2548c278..931b73b1e363 100644 --- a/lldb/source/Target/StackFrameList.cpp +++ b/lldb/source/Target/StackFrameList.cpp @@ -936,3 +936,5 @@ size_t StackFrameList::GetStatus(Stream &strm, uint32_t first_frame, strm.IndentLess(); return num_frames_displayed; } + +void StackFrameList::ClearSelectedFrameIndex() { m_selected_frame_idx.reset(); } diff --git a/lldb/source/Target/TargetProperties.td b/lldb/source/Target/TargetProperties.td index 4aa9e046d607..656503bb8d22 100644 --- a/lldb/source/Target/TargetProperties.td +++ b/lldb/source/Target/TargetProperties.td @@ -5,8 +5,8 @@ let Definition = "target_experimental" in { Global, DefaultTrue, Desc<"If true, inject local variables explicitly into the expression text. This will fix symbol resolution when there are name collisions between ivars and local variables. But it can make expressions run much more slowly.">; def UseDIL : Property<"use-DIL", "Boolean">, - Global, DefaultFalse, - Desc<"If true, use the alternative DIL implementation for frame variable evaluation.">; + Global, DefaultTrue, + Desc<"If true, use the DIL implementation for frame variable evaluation.">; } let Definition = "target" in { diff --git a/lldb/source/ValueObject/DILEval.cpp b/lldb/source/ValueObject/DILEval.cpp index fd3f9f872460..6f28434c646c 100644 --- a/lldb/source/ValueObject/DILEval.cpp +++ b/lldb/source/ValueObject/DILEval.cpp @@ -303,7 +303,7 @@ Interpreter::Visit(const MemberOfNode *node) { } } - if (field_obj && field_obj->GetName() == node->GetFieldName()) { + if (field_obj) { if (m_use_dynamic != lldb::eNoDynamicValues) { lldb::ValueObjectSP dynamic_val_sp = field_obj->GetDynamicValue(m_use_dynamic); diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index b985292ccee4..1dc73205a0eb 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -47,6 +47,8 @@ enum class RecurKind { FMul, ///< Product of floats. FMin, ///< FP min implemented in terms of select(cmp()). FMax, ///< FP max implemented in terms of select(cmp()). + FMinNum, ///< FP min with llvm.minnum semantics including NaNs. + FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs. FMinimum, ///< FP min with llvm.minimum semantics FMaximum, ///< FP max with llvm.maximum semantics FMinimumNum, ///< FP min with llvm.minimumnum semantics @@ -250,6 +252,7 @@ public: /// Returns true if the recurrence kind is a floating-point min/max kind. static bool isFPMinMaxRecurrenceKind(RecurKind Kind) { return Kind == RecurKind::FMin || Kind == RecurKind::FMax || + Kind == RecurKind::FMinNum || Kind == RecurKind::FMaxNum || Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum; } diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index ebb257ab3382..cfae75d09342 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -362,6 +362,7 @@ enum { ELFOSABI_FENIXOS = 16, // FenixOS ELFOSABI_CLOUDABI = 17, // Nuxi CloudABI ELFOSABI_CUDA = 51, // NVIDIA CUDA architecture. + ELFOSABI_CUDA_V2 = 41, // NVIDIA CUDA architecture. ELFOSABI_FIRST_ARCH = 64, // First architecture-specific OS ABI ELFOSABI_AMDGPU_HSA = 64, // AMD HSA runtime ELFOSABI_AMDGPU_PAL = 65, // AMD PAL runtime @@ -385,6 +386,12 @@ enum { ELFABIVERSION_AMDGPU_HSA_V6 = 4, }; +// CUDA OS ABI Version identification. +enum { + ELFABIVERSION_CUDA_V1 = 7, + ELFABIVERSION_CUDA_V2 = 8, +}; + #define ELF_RELOC(name, value) name = value, // X86_64 relocations. @@ -921,9 +928,15 @@ enum { // NVPTX specific e_flags. enum : unsigned { - // Processor selection mask for EF_CUDA_SM* values. + // Processor selection mask for EF_CUDA_SM* values prior to blackwell. EF_CUDA_SM = 0xff, + // Processor selection mask for EF_CUDA_SM* values following blackwell. + EF_CUDA_SM_MASK = 0xff00, + + // Processor selection mask for EF_CUDA_SM* values following blackwell. + EF_CUDA_SM_OFFSET = 8, + // SM based processor values. EF_CUDA_SM20 = 0x14, EF_CUDA_SM21 = 0x15, @@ -943,9 +956,15 @@ enum : unsigned { EF_CUDA_SM80 = 0x50, EF_CUDA_SM86 = 0x56, EF_CUDA_SM87 = 0x57, + EF_CUDA_SM88 = 0x58, EF_CUDA_SM89 = 0x59, - // The sm_90a variant uses the same machine flag. EF_CUDA_SM90 = 0x5a, + EF_CUDA_SM100 = 0x64, + EF_CUDA_SM101 = 0x65, + EF_CUDA_SM103 = 0x67, + EF_CUDA_SM110 = 0x6e, + EF_CUDA_SM120 = 0x78, + EF_CUDA_SM121 = 0x79, // Unified texture binding is enabled. EF_CUDA_TEXMODE_UNIFIED = 0x100, @@ -954,12 +973,15 @@ enum : unsigned { // The target is using 64-bit addressing. EF_CUDA_64BIT_ADDRESS = 0x400, // Set when using the sm_90a processor. - EF_CUDA_ACCELERATORS = 0x800, + EF_CUDA_ACCELERATORS_V1 = 0x800, // Undocumented software feature. EF_CUDA_SW_FLAG_V2 = 0x1000, // Virtual processor selection mask for EF_CUDA_VIRTUAL_SM* values. EF_CUDA_VIRTUAL_SM = 0xff0000, + + // Set when using an accelerator variant like sm_100a. + EF_CUDA_ACCELERATORS = 0x8, }; // ELF Relocation types for BPF diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index 938d71dd030e..9e3d9196cc18 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -323,10 +323,11 @@ public: const MachineFunction *getParent() const { return xParent; } MachineFunction *getParent() { return xParent; } - /// Returns true if the original IR terminator is an `indirectbr`. This - /// typically corresponds to a `goto` in C, rather than jump tables. - bool terminatorIsComputedGoto() const { - return back().isIndirectBranch() && + /// Returns true if the original IR terminator is an `indirectbr` with + /// successor blocks. This typically corresponds to a `goto` in C, rather than + /// jump tables. + bool terminatorIsComputedGotoWithSuccessors() const { + return back().isIndirectBranch() && !succ_empty() && llvm::all_of(successors(), [](const MachineBasicBlock *Succ) { return Succ->isIRBlockAddressTaken(); }); diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h index 646588a2a92a..9104e93ed978 100644 --- a/llvm/include/llvm/CodeGen/MachineOperand.h +++ b/llvm/include/llvm/CodeGen/MachineOperand.h @@ -788,6 +788,9 @@ public: LLVM_ABI void ChangeToBA(const BlockAddress *BA, int64_t Offset, unsigned TargetFlags = 0); + /// ChangeToCPI - Replace this operand with a new constant pool index operand. + LLVM_ABI void ChangeToCPI(unsigned Idx, int Offset, unsigned TargetFlags = 0); + /// ChangeToMCSymbol - Replace this operand with a new MC symbol operand. LLVM_ABI void ChangeToMCSymbol(MCSymbol *Sym, unsigned TargetFlags = 0); diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index b5b83c7ff116..4d6e34bf4e3a 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -1730,6 +1730,17 @@ public: return true; } + /// Return true if it's safe to move a machine instruction. + /// This allows the backend to prevent certain special instruction + /// sequences from being broken by instruction motion in optimization + /// passes. + /// By default, this returns true for every instruction. + virtual bool isSafeToMove(const MachineInstr &MI, + const MachineBasicBlock *MBB, + const MachineFunction &MF) const { + return true; + } + /// Test if the given instruction should be considered a scheduling boundary. /// This primarily includes labels and terminators. virtual bool isSchedulingBoundary(const MachineInstr &MI, diff --git a/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h b/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h index 8aac9d5b49db..448a6e913eb8 100644 --- a/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h +++ b/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h @@ -135,6 +135,12 @@ public: if (Mode.Input != DenormalMode::IEEE) return {Invalid, fcAllFlags, fcAllFlags}; + auto ExactClass = [IsFabs, Src](FPClassTest Mask) { + if (IsFabs) + Mask = llvm::inverse_fabs(Mask); + return exactClass(Src, Mask); + }; + switch (Pred) { case FCmpInst::FCMP_OEQ: // Match x == 0.0 return exactClass(Src, fcZero); @@ -151,26 +157,24 @@ public: case FCmpInst::FCMP_UNO: return exactClass(Src, fcNan); case FCmpInst::FCMP_OGT: // x > 0 - return exactClass(Src, fcPosSubnormal | fcPosNormal | fcPosInf); + return ExactClass(fcPosSubnormal | fcPosNormal | fcPosInf); case FCmpInst::FCMP_UGT: // isnan(x) || x > 0 - return exactClass(Src, fcPosSubnormal | fcPosNormal | fcPosInf | fcNan); + return ExactClass(fcPosSubnormal | fcPosNormal | fcPosInf | fcNan); case FCmpInst::FCMP_OGE: // x >= 0 - return exactClass(Src, fcPositive | fcNegZero); + return ExactClass(fcPositive | fcNegZero); case FCmpInst::FCMP_UGE: // isnan(x) || x >= 0 - return exactClass(Src, fcPositive | fcNegZero | fcNan); + return ExactClass(fcPositive | fcNegZero | fcNan); case FCmpInst::FCMP_OLT: // x < 0 - return exactClass(Src, fcNegSubnormal | fcNegNormal | fcNegInf); + return ExactClass(fcNegSubnormal | fcNegNormal | fcNegInf); case FCmpInst::FCMP_ULT: // isnan(x) || x < 0 - return exactClass(Src, fcNegSubnormal | fcNegNormal | fcNegInf | fcNan); + return ExactClass(fcNegSubnormal | fcNegNormal | fcNegInf | fcNan); case FCmpInst::FCMP_OLE: // x <= 0 - return exactClass(Src, fcNegative | fcPosZero); + return ExactClass(fcNegative | fcPosZero); case FCmpInst::FCMP_ULE: // isnan(x) || x <= 0 - return exactClass(Src, fcNegative | fcPosZero | fcNan); + return ExactClass(fcNegative | fcPosZero | fcNan); default: llvm_unreachable("all compare types are handled"); } - - return {Invalid, fcAllFlags, fcAllFlags}; } const bool IsDenormalRHS = (OrigClass & fcSubnormal) == OrigClass; diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td index a6254eafa490..9b081e9d6544 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.td +++ b/llvm/include/llvm/IR/RuntimeLibcalls.td @@ -2120,7 +2120,7 @@ defvar X86CommonLibcalls = ); defvar Windows32DivRemMulCalls = - LibcallImpls<(add WindowsDivRemMulLibcalls), + LibcallsWithCC<(add WindowsDivRemMulLibcalls), X86_STDCALL, RuntimeLibcallPredicate<"TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment()">>; def X86_32SystemLibrary diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index 5a8ec17dae1c..22a493a8df01 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -489,6 +489,10 @@ public: /// \param Name - The symbol name, which must be unique across all symbols. LLVM_ABI MCSymbol *getOrCreateSymbol(const Twine &Name); + /// Variant of getOrCreateSymbol that handles backslash-escaped symbols. + /// For example, parse "a\"b\\" as a"\. + LLVM_ABI MCSymbol *parseSymbol(const Twine &Name); + /// Gets a symbol that will be defined to the final stack offset of a local /// variable after codegen. /// diff --git a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h index 3a7ca1a69ab8..cae2fbcac1fe 100644 --- a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h +++ b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h @@ -136,6 +136,18 @@ public: ArrayRef<uint8_t> Bytes, uint64_t Address, raw_ostream &CStream) const = 0; + /// Returns the disassembly of an instruction bundle for VLIW architectures + /// like Hexagon. + /// + /// \param Instr - An MCInst to populate with the contents of + /// the Bundle with sub-instructions encoded as Inst operands. + virtual DecodeStatus getInstructionBundle(MCInst &Instr, uint64_t &Size, + ArrayRef<uint8_t> Bytes, + uint64_t Address, + raw_ostream &CStream) const { + return Fail; + } + /// Used to perform separate target specific disassembly for a particular /// symbol. May parse any prelude that precedes instructions after the /// start of a symbol, or the entire symbol. diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h index cb9bd5c600d5..e3f44a08db64 100644 --- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h @@ -279,6 +279,9 @@ public: /// Res to the identifier contents. virtual bool parseIdentifier(StringRef &Res) = 0; + /// Parse identifier and get or create symbol for it. + bool parseSymbol(MCSymbol *&Res); + /// Parse up to the end of statement and return the contents from the /// current token until the end of the statement; the current token on exit /// will be either the EndOfStatement or EOF. diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index 64b13972bfca..9daaebf7e793 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -298,8 +298,8 @@ public: /// data. class MCEncodedFragment : public MCFragment { uint8_t BundlePadding = 0; - uint32_t ContentStart = 0; - uint32_t ContentEnd = 0; + uint32_t ContentSize = 0; + uint64_t ContentStart = 0; uint32_t FixupStart = 0; uint32_t FixupEnd = 0; @@ -360,22 +360,23 @@ public: // Content-related functions manage parent's storage using ContentStart and // ContentSize. - void clearContents() { ContentEnd = ContentStart; } + void clearContents() { ContentSize = 0; } // Get a SmallVector reference. The caller should call doneAppending to update - // `ContentEnd`. + // `ContentSize`. SmallVectorImpl<char> &getContentsForAppending() { SmallVectorImpl<char> &S = getParent()->ContentStorage; - if (LLVM_UNLIKELY(ContentEnd != S.size())) { + if (LLVM_UNLIKELY(ContentStart + ContentSize != S.size())) { // Move the elements to the end. Reserve space to avoid invalidating // S.begin()+I for `append`. - auto Size = ContentEnd - ContentStart; auto I = std::exchange(ContentStart, S.size()); - S.reserve(S.size() + Size); - S.append(S.begin() + I, S.begin() + I + Size); + S.reserve(S.size() + ContentSize); + S.append(S.begin() + I, S.begin() + I + ContentSize); } return S; } - void doneAppending() { ContentEnd = getParent()->ContentStorage.size(); } + void doneAppending() { + ContentSize = getParent()->ContentStorage.size() - ContentStart; + } void appendContents(ArrayRef<char> Contents) { getContentsForAppending().append(Contents.begin(), Contents.end()); doneAppending(); @@ -387,11 +388,11 @@ public: LLVM_ABI void setContents(ArrayRef<char> Contents); MutableArrayRef<char> getContents() { return MutableArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); + .slice(ContentStart, ContentSize); } ArrayRef<char> getContents() const { return ArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); + .slice(ContentStart, ContentSize); } // Fixup-related functions manage parent's storage using FixupStart and @@ -409,7 +410,7 @@ public: .slice(FixupStart, FixupEnd - FixupStart); } - size_t getSize() const { return ContentEnd - ContentStart; } + size_t getSize() const { return ContentSize; } }; /// Fragment for data and encoded instructions. diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index 103686884e70..30a9dd35f624 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -1479,6 +1479,7 @@ template <class ELFT> Triple::OSType ELFObjectFile<ELFT>::getOS() const { case ELF::ELFOSABI_OPENBSD: return Triple::OpenBSD; case ELF::ELFOSABI_CUDA: + case ELF::ELFOSABI_CUDA_V2: return Triple::CUDA; case ELF::ELFOSABI_AMDGPU_HSA: return Triple::AMDHSA; diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 39f74beca082..8be5de3bf356 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -941,10 +941,30 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr( m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) || match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value())); }; - if (isIntMinMaxRecurrenceKind(Kind) || - (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind))) + if (isIntMinMaxRecurrenceKind(Kind)) return isMinMaxPattern(I, Kind, Prev); - else if (isFMulAddIntrinsic(I)) + if (isFPMinMaxRecurrenceKind(Kind)) { + InstDesc Res = isMinMaxPattern(I, Kind, Prev); + if (!Res.isRecurrence()) + return InstDesc(false, I); + if (HasRequiredFMF()) + return Res; + // We may be able to vectorize FMax/FMin reductions using maxnum/minnum + // intrinsics with extra checks ensuring the vector loop handles only + // non-NaN inputs. + if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) { + assert(Kind == RecurKind::FMax && + "unexpected recurrence kind for maxnum"); + return InstDesc(I, RecurKind::FMaxNum); + } + if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) { + assert(Kind == RecurKind::FMin && + "unexpected recurrence kind for minnum"); + return InstDesc(I, RecurKind::FMinNum); + } + return InstDesc(false, I); + } + if (isFMulAddIntrinsic(I)) return InstDesc(Kind == RecurKind::FMulAdd, I, I->hasAllowReassoc() ? nullptr : I); return InstDesc(false, I); diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index 393f2648de3c..fcc2cf2f7e8e 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -382,7 +382,10 @@ bool llvm::isDereferenceableAndAlignedInLoop( if (Offset->getAPInt().urem(Alignment.value()) != 0) return false; - AccessSize = MaxPtrDiff + Offset->getAPInt(); + bool Overflow = false; + AccessSize = MaxPtrDiff.uadd_ov(Offset->getAPInt(), Overflow); + if (Overflow) + return false; AccessSizeSCEV = SE.getAddExpr(PtrDiff, Offset); Base = NewBase->getValue(); } else diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 24adfa346c64..221468a2d1a8 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15093,15 +15093,20 @@ void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) { return; } + // Implication checks are quadratic in the number of predicates. Stop doing + // them if there are many predicates, as they should be too expensive to use + // anyway at that point. + bool CheckImplies = Preds.size() < 16; + // Only add predicate if it is not already implied by this union predicate. - if (implies(N, SE)) + if (CheckImplies && implies(N, SE)) return; // Build a new vector containing the current predicates, except the ones that // are implied by the new predicate N. SmallVector<const SCEVPredicate *> PrunedPreds; for (auto *P : Preds) { - if (N->implies(P, SE)) + if (CheckImplies && N->implies(P, SE)) continue; PrunedPreds.push_back(P); } diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp index 3b3e7a418feb..f32a18833d16 100644 --- a/llvm/lib/CodeGen/BranchFolding.cpp +++ b/llvm/lib/CodeGen/BranchFolding.cpp @@ -1971,6 +1971,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) { MachineBasicBlock::iterator FIB = FBB->begin(); MachineBasicBlock::iterator TIE = TBB->end(); MachineBasicBlock::iterator FIE = FBB->end(); + MachineFunction &MF = *TBB->getParent(); while (TIB != TIE && FIB != FIE) { // Skip dbg_value instructions. These do not count. TIB = skipDebugInstructionsForward(TIB, TIE, false); @@ -1985,6 +1986,10 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) { // Hard to reason about register liveness with predicated instruction. break; + if (!TII->isSafeToMove(*TIB, TBB, MF)) + // Don't hoist the instruction if it isn't safe to move. + break; + bool IsSafe = true; for (MachineOperand &MO : TIB->operands()) { // Don't attempt to hoist instructions with register masks. diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 9bbb89e37865..3d1408256df8 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -2096,6 +2096,10 @@ static bool isRemOfLoopIncrementWithLoopInvariant( if (!L->isLoopInvariant(RemAmt)) return false; + // Only works if the AddOffset is a loop invaraint + if (AddOffset && !L->isLoopInvariant(AddOffset)) + return false; + // Is the PHI a loop increment? auto LoopIncrInfo = getIVIncrement(PN, LI); if (!LoopIncrInfo) diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp index 742de1101faa..620767a39b91 100644 --- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp +++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp @@ -937,16 +937,6 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) { if (CopyOperands) { Register RegSrc = CopyOperands->Source->getReg(); Register RegDef = CopyOperands->Destination->getReg(); - // It's possible that the previous transformations have resulted in a - // no-op register move (i.e. one where source and destination registers - // are the same and are not referring to a reserved register). If so, - // delete it. - if (RegSrc == RegDef && !MRI->isReserved(RegSrc)) { - MI.eraseFromParent(); - NumDeletes++; - Changed = true; - continue; - } if (!TRI->regsOverlap(RegDef, RegSrc)) { // Copy is now a candidate for deletion. diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp index 0d251697f256..f09855c43af6 100644 --- a/llvm/lib/CodeGen/MachineOperand.cpp +++ b/llvm/lib/CodeGen/MachineOperand.cpp @@ -218,6 +218,19 @@ void MachineOperand::ChangeToBA(const BlockAddress *BA, int64_t Offset, setTargetFlags(TargetFlags); } +void MachineOperand::ChangeToCPI(unsigned Idx, int Offset, + unsigned TargetFlags) { + assert((!isReg() || !isTied()) && + "Cannot change a tied operand into a constant pool index"); + + removeRegFromUses(); + + OpKind = MO_ConstantPoolIndex; + setIndex(Idx); + setOffset(Offset); + setTargetFlags(TargetFlags); +} + void MachineOperand::ChangeToMCSymbol(MCSymbol *Sym, unsigned TargetFlags) { assert((!isReg() || !isTied()) && "Cannot change a tied operand into an MCSymbol"); diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index b38a4d1c55af..0e7cb0c980d4 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -110,6 +110,7 @@ STATISTIC(NumFailZeroMII, "Pipeliner abort due to zero MII"); STATISTIC(NumFailNoSchedule, "Pipeliner abort due to no schedule found"); STATISTIC(NumFailZeroStage, "Pipeliner abort due to zero stage"); STATISTIC(NumFailLargeMaxStage, "Pipeliner abort due to too many stages"); +STATISTIC(NumFailTooManyStores, "Pipeliner abort due to too many stores"); /// A command line option to turn software pipelining on or off. static cl::opt<bool> EnableSWP("enable-pipeliner", cl::Hidden, cl::init(true), @@ -193,6 +194,13 @@ static cl::opt<bool> MVECodeGen("pipeliner-mve-cg", cl::Hidden, cl::init(false), cl::desc("Use the MVE code generator for software pipelining")); +/// A command line argument to limit the number of store instructions in the +/// target basic block. +static cl::opt<unsigned> SwpMaxNumStores( + "pipeliner-max-num-stores", + cl::desc("Maximum number of stores allwed in the target loop."), cl::Hidden, + cl::init(200)); + namespace llvm { // A command line option to enable the CopyToPhi DAG mutation. @@ -544,6 +552,23 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) { return false; } + unsigned NumStores = 0; + for (MachineInstr &MI : *L.getHeader()) + if (MI.mayStore()) + ++NumStores; + if (NumStores > SwpMaxNumStores) { + LLVM_DEBUG(dbgs() << "Too many stores\n"); + NumFailTooManyStores++; + ORE->emit([&]() { + return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "canPipelineLoop", + L.getStartLoc(), L.getHeader()) + << "Too many store instructions in the loop: " + << ore::NV("NumStores", NumStores) << " > " + << ore::NV("SwpMaxNumStores", SwpMaxNumStores) << "."; + }); + return false; + } + // Remove any subregisters from inputs to phi nodes. preprocessPhiNodes(*L.getHeader()); return true; @@ -4279,8 +4304,8 @@ void LoopCarriedEdges::modifySUnits(std::vector<SUnit> &SUnits, !TII->isGlobalMemoryObject(FromMI) && !TII->isGlobalMemoryObject(ToMI) && !isSuccOrder(From, To)) { SDep Pred = Dep; - Pred.setSUnit(Src); - Dst->addPred(Pred); + Pred.setSUnit(From); + To->addPred(Pred); } } } diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 8de2c48581a1..9fa96e737296 100644 --- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -587,12 +587,14 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { break; case Intrinsic::exp: case Intrinsic::exp2: + case Intrinsic::log: Changed |= forEachCall(F, [&](CallInst *CI) { Type *Ty = CI->getArgOperand(0)->getType(); if (!isa<ScalableVectorType>(Ty)) return false; const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering(); unsigned Op = TL->IntrinsicIDToISD(F.getIntrinsicID()); + assert(Op != ISD::DELETED_NODE && "unsupported intrinsic"); if (!TL->isOperationExpand(Op, EVT::getEVT(Ty))) return false; return lowerUnaryVectorIntrinsicAsLoop(M, CI); diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 23812d795f5f..91fd2d843f44 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -16717,7 +16717,8 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { // Fold freeze(op(x, ...)) -> op(freeze(x), ...). // Try to push freeze through instructions that propagate but don't produce // poison as far as possible. If an operand of freeze follows three - // conditions 1) one-use, and 2) does not produce poison then push + // conditions 1) one-use, 2) does not produce poison, and 3) has all but one + // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push // the freeze through to the operands that are not guaranteed non-poison. // NOTE: we will strip poison-generating flags, so ignore them here. if (DAG.canCreateUndefOrPoison(N0, /*PoisonOnly*/ false, @@ -16725,6 +16726,18 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { N0->getNumValues() != 1 || !N0->hasOneUse()) return SDValue(); + // TOOD: we should always allow multiple operands, however this increases the + // likelihood of infinite loops due to the ReplaceAllUsesOfValueWith call + // below causing later nodes that share frozen operands to fold again and no + // longer being able to confirm other operands are not poison due to recursion + // depth limits on isGuaranteedNotToBeUndefOrPoison. + bool AllowMultipleMaybePoisonOperands = + N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC || + N0.getOpcode() == ISD::BUILD_VECTOR || + N0.getOpcode() == ISD::BUILD_PAIR || + N0.getOpcode() == ISD::VECTOR_SHUFFLE || + N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL; + // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all // ones" or "constant" into something that depends on FrozenUndef. We can // instead pick undef values to keep those properties, while at the same time @@ -16757,6 +16770,10 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) { MaybePoisonOperandNumbers.push_back(OpNo); if (!HadMaybePoisonOperands) continue; + if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) { + // Multiple maybe-poison ops when not allowed - bail out. + return SDValue(); + } } // NOTE: the whole op may be not guaranteed to not be undef or poison because // it could create undef or poison due to it's poison-generating flags. diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 2cad36eff9c8..fe357106bda6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -1551,6 +1551,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; + case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break; case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; case ISD::STRICT_FMINNUM: case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; @@ -1966,6 +1967,13 @@ void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); } +void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo, + SDValue &Hi) { + // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves. + SDLoc dl(N); + GetExpandedFloat(N->getOperand(0), Lo, Hi); +} + void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, SDValue &Hi) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 9b537248d4ab..4eaa79890e00 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -677,6 +677,7 @@ private: SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo = {}); // clang-format off + void ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FACOS (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FASIN (SDNode *N, SDValue &Lo, SDValue &Hi); diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp index a88c57fdc165..8cbdadd97981 100644 --- a/llvm/lib/CodeGen/TailDuplicator.cpp +++ b/llvm/lib/CodeGen/TailDuplicator.cpp @@ -604,12 +604,23 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple, bool HasComputedGoto = false; if (!TailBB.empty()) { HasIndirectbr = TailBB.back().isIndirectBranch(); - HasComputedGoto = TailBB.terminatorIsComputedGoto(); + HasComputedGoto = TailBB.terminatorIsComputedGotoWithSuccessors(); } if (HasIndirectbr && PreRegAlloc) MaxDuplicateCount = TailDupIndirectBranchSize; + // Allow higher limits when the block has computed-gotos and running after + // register allocation. NB. This basically unfactors computed gotos that were + // factored early on in the compilation process to speed up edge based data + // flow. If we do not unfactor them again, it can seriously pessimize code + // with many computed jumps in the source code, such as interpreters. + // Therefore we do not restrict the computed gotos. + bool DupComputedGotoLate = + HasComputedGoto && MF->getTarget().getTargetTriple().isOSDarwin(); + if (DupComputedGotoLate && !PreRegAlloc) + MaxDuplicateCount = std::max(MaxDuplicateCount, 10u); + // Check the instructions in the block to determine whether tail-duplication // is invalid or unlikely to be profitable. unsigned InstrCount = 0; @@ -663,12 +674,10 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple, // Duplicating a BB which has both multiple predecessors and successors will // may cause huge amount of PHI nodes. If we want to remove this limitation, // we have to address https://github.com/llvm/llvm-project/issues/78578. - // NB. This basically unfactors computed gotos that were factored early on in - // the compilation process to speed up edge based data flow. If we do not - // unfactor them again, it can seriously pessimize code with many computed - // jumps in the source code, such as interpreters. Therefore we do not - // restrict the computed gotos. - if (!HasComputedGoto && TailBB.pred_size() > TailDupPredSize && + bool CheckSuccessorAndPredecessorSize = + DupComputedGotoLate ? PreRegAlloc : !HasComputedGoto; + if (CheckSuccessorAndPredecessorSize && + TailBB.pred_size() > TailDupPredSize && TailBB.succ_size() > TailDupSuccSize) { // If TailBB or any of its successors contains a phi, we may have to add a // large number of additional phis with additional incoming values. diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index 6feeb19bb858..ca2a57e9b7b2 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -806,7 +806,17 @@ void TargetLoweringBase::initActions() { ISD::SDIVFIX, ISD::SDIVFIXSAT, ISD::UDIVFIX, ISD::UDIVFIXSAT, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, - ISD::IS_FPCLASS}, + ISD::IS_FPCLASS, ISD::FCBRT, + ISD::FLOG, ISD::FLOG2, + ISD::FLOG10, ISD::FEXP, + ISD::FEXP2, ISD::FEXP10, + ISD::FFLOOR, ISD::FNEARBYINT, + ISD::FCEIL, ISD::FRINT, + ISD::FTRUNC, ISD::FROUNDEVEN, + ISD::FTAN, ISD::FACOS, + ISD::FASIN, ISD::FATAN, + ISD::FCOSH, ISD::FSINH, + ISD::FTANH, ISD::FATAN2}, VT, Expand); // Overflow operations default to expand @@ -852,13 +862,12 @@ void TargetLoweringBase::initActions() { // These operations default to expand for vector types. if (VT.isVector()) - setOperationAction( - {ISD::FCOPYSIGN, ISD::SIGN_EXTEND_INREG, ISD::ANY_EXTEND_VECTOR_INREG, - ISD::SIGN_EXTEND_VECTOR_INREG, ISD::ZERO_EXTEND_VECTOR_INREG, - ISD::SPLAT_VECTOR, ISD::LRINT, ISD::LLRINT, ISD::LROUND, - ISD::LLROUND, ISD::FTAN, ISD::FACOS, ISD::FASIN, ISD::FATAN, - ISD::FCOSH, ISD::FSINH, ISD::FTANH, ISD::FATAN2}, - VT, Expand); + setOperationAction({ISD::FCOPYSIGN, ISD::SIGN_EXTEND_INREG, + ISD::ANY_EXTEND_VECTOR_INREG, + ISD::SIGN_EXTEND_VECTOR_INREG, + ISD::ZERO_EXTEND_VECTOR_INREG, ISD::SPLAT_VECTOR, + ISD::LRINT, ISD::LLRINT, ISD::LROUND, ISD::LLROUND}, + VT, Expand); // Constrained floating-point operations default to expand. #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ @@ -914,15 +923,6 @@ void TargetLoweringBase::initActions() { {MVT::bf16, MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128}, Expand); - // These library functions default to expand. - setOperationAction({ISD::FCBRT, ISD::FLOG, ISD::FLOG2, ISD::FLOG10, - ISD::FEXP, ISD::FEXP2, ISD::FEXP10, ISD::FFLOOR, - ISD::FNEARBYINT, ISD::FCEIL, ISD::FRINT, ISD::FTRUNC, - ISD::FROUNDEVEN, ISD::FTAN, ISD::FACOS, ISD::FASIN, - ISD::FATAN, ISD::FCOSH, ISD::FSINH, ISD::FTANH, - ISD::FATAN2}, - {MVT::f32, MVT::f64, MVT::f128}, Expand); - // Insert custom handling default for llvm.canonicalize.*. setOperationAction(ISD::FCANONICALIZE, {MVT::f16, MVT::f32, MVT::f64, MVT::f128}, Expand); @@ -1922,6 +1922,8 @@ int TargetLoweringBase::IntrinsicIDToISD(Intrinsic::ID ID) const { return ISD::FEXP; case Intrinsic::exp2: return ISD::FEXP2; + case Intrinsic::log: + return ISD::FLOG; default: return ISD::DELETED_NODE; } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 8004077b9266..9afbefb8c08e 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3185,12 +3185,6 @@ void Verifier::visitFunction(const Function &F) { CheckDI(SP->describes(&F), "!dbg attachment points at wrong subprogram for function", N, &F, &I, DL, Scope, SP); - - if (DL->getAtomGroup()) - CheckDI(DL->getScope()->getSubprogram()->getKeyInstructionsEnabled(), - "DbgLoc uses atomGroup but DISubprogram doesn't have Key " - "Instructions enabled", - DL, DL->getScope()->getSubprogram()); }; for (auto &BB : F) for (auto &I : BB) { @@ -5492,6 +5486,15 @@ void Verifier::visitInstruction(Instruction &I) { if (MDNode *N = I.getDebugLoc().getAsMDNode()) { CheckDI(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N); visitMDNode(*N, AreDebugLocsAllowed::Yes); + + if (auto *DL = dyn_cast<DILocation>(N)) { + if (DL->getAtomGroup()) { + CheckDI(DL->getScope()->getSubprogram()->getKeyInstructionsEnabled(), + "DbgLoc uses atomGroup but DISubprogram doesn't have Key " + "Instructions enabled", + DL, DL->getScope()->getSubprogram()); + } + } } if (auto *DII = dyn_cast<DbgVariableIntrinsic>(&I)) { diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp index 890184db1d1e..1625455e38e0 100644 --- a/llvm/lib/MC/MCContext.cpp +++ b/llvm/lib/MC/MCContext.cpp @@ -217,27 +217,6 @@ MCDataFragment *MCContext::allocInitialFragment(MCSection &Sec) { MCSymbol *MCContext::getOrCreateSymbol(const Twine &Name) { SmallString<128> NameSV; StringRef NameRef = Name.toStringRef(NameSV); - if (NameRef.contains('\\')) { - NameSV = NameRef; - size_t S = 0; - // Support escaped \\ and \" as in GNU Assembler. GAS issues a warning for - // other characters following \\, which we do not implement due to code - // structure. - for (size_t I = 0, E = NameSV.size(); I != E; ++I) { - char C = NameSV[I]; - if (C == '\\' && I + 1 != E) { - switch (NameSV[I + 1]) { - case '"': - case '\\': - C = NameSV[++I]; - break; - } - } - NameSV[S++] = C; - } - NameSV.resize(S); - NameRef = NameSV; - } assert(!NameRef.empty() && "Normal symbols cannot be unnamed!"); @@ -258,6 +237,34 @@ MCSymbol *MCContext::getOrCreateSymbol(const Twine &Name) { return Entry.second.Symbol; } +MCSymbol *MCContext::parseSymbol(const Twine &Name) { + SmallString<128> SV; + StringRef NameRef = Name.toStringRef(SV); + if (NameRef.contains('\\')) { + SV = NameRef; + size_t S = 0; + // Support escaped \\ and \" as in GNU Assembler. GAS issues a warning for + // other characters following \\, which we do not implement due to code + // structure. + for (size_t I = 0, E = SV.size(); I != E; ++I) { + char C = SV[I]; + if (C == '\\' && I + 1 != E) { + switch (SV[I + 1]) { + case '"': + case '\\': + C = SV[++I]; + break; + } + } + SV[S++] = C; + } + SV.resize(S); + NameRef = SV; + } + + return getOrCreateSymbol(NameRef); +} + MCSymbol *MCContext::getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx) { return getOrCreateSymbol(MAI->getPrivateGlobalPrefix() + FuncName + diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp index a19659174409..a877eb7938eb 100644 --- a/llvm/lib/MC/MCExpr.cpp +++ b/llvm/lib/MC/MCExpr.cpp @@ -361,6 +361,9 @@ static void attemptToFoldSymbolOffsetDifference(const MCAssembler *Asm, if (BBeforeRelax && AAfterRelax) return; } + const auto *RF = dyn_cast<MCRelaxableFragment>(F); + if (RF && RF->isLinkerRelaxable()) + return; if (&*F == FA) { // If FA and FB belong to the same subsection, the loop will find FA and // we can resolve the difference. diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp index c59fabe5df1b..28cddb939646 100644 --- a/llvm/lib/MC/MCFragment.cpp +++ b/llvm/lib/MC/MCFragment.cpp @@ -70,6 +70,8 @@ LLVM_DUMP_METHOD void MCFragment::dump() const { OS << "\n Fixup @" << F.getOffset() << " Value:"; F.getValue()->print(OS, nullptr); OS << " Kind:" << F.getKind(); + if (F.isLinkerRelaxable()) + OS << " LinkerRelaxable"; } }; @@ -113,6 +115,8 @@ LLVM_DUMP_METHOD void MCFragment::dump() const { } case MCFragment::FT_Relaxable: { const auto *F = cast<MCRelaxableFragment>(this); + if (F->isLinkerRelaxable()) + OS << " LinkerRelaxable"; OS << " Size:" << F->getContents().size() << ' '; F->getInst().dump_pretty(OS); printFixups(F->getFixups()); diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 44a82f75576b..d9c39bbedf37 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -408,6 +408,13 @@ void MCObjectStreamer::emitInstToFragment(const MCInst &Inst, Inst, IF->getContentsForAppending(), Fixups, STI); IF->doneAppending(); IF->appendFixups(Fixups); + + for (auto &Fixup : Fixups) { + if (Fixup.isLinkerRelaxable()) { + IF->setLinkerRelaxable(); + getCurrentSectionOnly()->setLinkerRelaxable(); + } + } } #ifndef NDEBUG diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp index 9fd6c05a846d..929051c4f0c1 100644 --- a/llvm/lib/MC/MCParser/AsmParser.cpp +++ b/llvm/lib/MC/MCParser/AsmParser.cpp @@ -1222,8 +1222,8 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc, MCSymbol *Sym = getContext().getInlineAsmLabel(SymbolName); if (!Sym) - Sym = getContext().getOrCreateSymbol(MAI.isHLASM() ? SymbolName.upper() - : SymbolName); + Sym = getContext().parseSymbol(MAI.isHLASM() ? SymbolName.upper() + : SymbolName); // If this is an absolute variable reference, substitute it now to preserve // semantics in the face of reassignment. @@ -1854,7 +1854,7 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info, RewrittenLabel); IDVal = RewrittenLabel; } - Sym = getContext().getOrCreateSymbol(IDVal); + Sym = getContext().parseSymbol(IDVal); } else Sym = Ctx.createDirectionalLocalSymbol(LocalLabelVal); // End of Labels should be treated as end of line for lexing @@ -3897,20 +3897,15 @@ bool AsmParser::parseDirectiveCVLoc() { /// ::= .cv_linetable FunctionId, FnStart, FnEnd bool AsmParser::parseDirectiveCVLinetable() { int64_t FunctionId; - StringRef FnStartName, FnEndName; + MCSymbol *FnStartSym, *FnEndSym; SMLoc Loc = getTok().getLoc(); if (parseCVFunctionId(FunctionId, ".cv_linetable") || parseComma() || parseTokenLoc(Loc) || - check(parseIdentifier(FnStartName), Loc, - "expected identifier in directive") || + check(parseSymbol(FnStartSym), Loc, "expected identifier in directive") || parseComma() || parseTokenLoc(Loc) || - check(parseIdentifier(FnEndName), Loc, - "expected identifier in directive")) + check(parseSymbol(FnEndSym), Loc, "expected identifier in directive")) return true; - MCSymbol *FnStartSym = getContext().getOrCreateSymbol(FnStartName); - MCSymbol *FnEndSym = getContext().getOrCreateSymbol(FnEndName); - getStreamer().emitCVLinetableDirective(FunctionId, FnStartSym, FnEndSym); return false; } @@ -3919,7 +3914,7 @@ bool AsmParser::parseDirectiveCVLinetable() { /// ::= .cv_inline_linetable PrimaryFunctionId FileId LineNum FnStart FnEnd bool AsmParser::parseDirectiveCVInlineLinetable() { int64_t PrimaryFunctionId, SourceFileId, SourceLineNum; - StringRef FnStartName, FnEndName; + MCSymbol *FnStartSym, *FnEndSym; SMLoc Loc = getTok().getLoc(); if (parseCVFunctionId(PrimaryFunctionId, ".cv_inline_linetable") || parseTokenLoc(Loc) || @@ -3929,16 +3924,14 @@ bool AsmParser::parseDirectiveCVInlineLinetable() { parseIntToken(SourceLineNum, "expected SourceLineNum") || check(SourceLineNum < 0, Loc, "Line number less than zero") || parseTokenLoc(Loc) || - check(parseIdentifier(FnStartName), Loc, "expected identifier") || + check(parseSymbol(FnStartSym), Loc, "expected identifier") || parseTokenLoc(Loc) || - check(parseIdentifier(FnEndName), Loc, "expected identifier")) + check(parseSymbol(FnEndSym), Loc, "expected identifier")) return true; if (parseEOL()) return true; - MCSymbol *FnStartSym = getContext().getOrCreateSymbol(FnStartName); - MCSymbol *FnEndSym = getContext().getOrCreateSymbol(FnEndName); getStreamer().emitCVInlineLinetableDirective(PrimaryFunctionId, SourceFileId, SourceLineNum, FnStartSym, FnEndSym); @@ -3959,16 +3952,14 @@ bool AsmParser::parseDirectiveCVDefRange() { std::vector<std::pair<const MCSymbol *, const MCSymbol *>> Ranges; while (getLexer().is(AsmToken::Identifier)) { Loc = getLexer().getLoc(); - StringRef GapStartName; - if (parseIdentifier(GapStartName)) + MCSymbol *GapStartSym; + if (parseSymbol(GapStartSym)) return Error(Loc, "expected identifier in directive"); - MCSymbol *GapStartSym = getContext().getOrCreateSymbol(GapStartName); Loc = getLexer().getLoc(); - StringRef GapEndName; - if (parseIdentifier(GapEndName)) + MCSymbol *GapEndSym; + if (parseSymbol(GapEndSym)) return Error(Loc, "expected identifier in directive"); - MCSymbol *GapEndSym = getContext().getOrCreateSymbol(GapEndName); Ranges.push_back({GapStartSym, GapEndSym}); } @@ -4105,12 +4096,11 @@ bool AsmParser::parseDirectiveCVFileChecksumOffset() { /// ::= .cv_fpo_data procsym bool AsmParser::parseDirectiveCVFPOData() { SMLoc DirLoc = getLexer().getLoc(); - StringRef ProcName; - if (parseIdentifier(ProcName)) + MCSymbol *ProcSym; + if (parseSymbol(ProcSym)) return TokError("expected symbol name"); if (parseEOL()) return true; - MCSymbol *ProcSym = getContext().getOrCreateSymbol(ProcName); getStreamer().emitCVFPOData(ProcSym, DirLoc); return false; } @@ -4329,15 +4319,12 @@ bool AsmParser::parseDirectiveCFIPersonalityOrLsda(bool IsPersonality) { if (Encoding == dwarf::DW_EH_PE_omit) return false; - StringRef Name; + MCSymbol *Sym; if (check(!isValidEncoding(Encoding), "unsupported encoding.") || parseComma() || - check(parseIdentifier(Name), "expected identifier in directive") || - parseEOL()) + check(parseSymbol(Sym), "expected identifier in directive") || parseEOL()) return true; - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - if (IsPersonality) getStreamer().emitCFIPersonality(Sym, Encoding); else @@ -4966,7 +4953,7 @@ bool AsmParser::parseDirectiveSymbolAttribute(MCSymbolAttr Attr) { if (discardLTOSymbol(Name)) return false; - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); + MCSymbol *Sym = getContext().parseSymbol(Name); // Assembler local symbols don't make any sense here, except for directives // that the symbol should be tagged. @@ -4988,13 +4975,10 @@ bool AsmParser::parseDirectiveComm(bool IsLocal) { return true; SMLoc IDLoc = getLexer().getLoc(); - StringRef Name; - if (parseIdentifier(Name)) + MCSymbol *Sym; + if (parseSymbol(Sym)) return TokError("expected identifier in directive"); - // Handle the identifier as the key symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - if (parseComma()) return true; @@ -5827,10 +5811,9 @@ bool AsmParser::parseDirectiveAddrsig() { } bool AsmParser::parseDirectiveAddrsigSym() { - StringRef Name; - if (check(parseIdentifier(Name), "expected identifier") || parseEOL()) + MCSymbol *Sym; + if (check(parseSymbol(Sym), "expected identifier") || parseEOL()) return true; - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); getStreamer().emitAddrsigSym(Sym); return false; } @@ -6230,7 +6213,7 @@ bool HLASMAsmParser::parseAsHLASMLabel(ParseStatementInfo &Info, return Error(LabelLoc, "Cannot have just a label for an HLASM inline asm statement"); - MCSymbol *Sym = getContext().getOrCreateSymbol( + MCSymbol *Sym = getContext().parseSymbol( getContext().getAsmInfo()->isHLASM() ? LabelVal.upper() : LabelVal); // Emit the label. @@ -6357,7 +6340,7 @@ bool parseAssignmentExpression(StringRef Name, bool allow_redef, Parser.getStreamer().emitValueToOffset(Value, 0, EqualLoc); return false; } else - Sym = Parser.getContext().getOrCreateSymbol(Name); + Sym = Parser.getContext().parseSymbol(Name); Sym->setRedefinable(allow_redef); diff --git a/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/llvm/lib/MC/MCParser/COFFAsmParser.cpp index 9fb17488a9e9..5dd79946d877 100644 --- a/llvm/lib/MC/MCParser/COFFAsmParser.cpp +++ b/llvm/lib/MC/MCParser/COFFAsmParser.cpp @@ -293,13 +293,11 @@ bool COFFAsmParser::parseDirectiveSymbolAttribute(StringRef Directive, SMLoc) { assert(Attr != MCSA_Invalid && "unexpected symbol attribute directive!"); if (getLexer().isNot(AsmToken::EndOfStatement)) { while (true) { - StringRef Name; + MCSymbol *Sym; - if (getParser().parseIdentifier(Name)) + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - getStreamer().emitSymbolAttribute(Sym, Attr); if (getLexer().is(AsmToken::EndOfStatement)) @@ -450,13 +448,11 @@ bool COFFAsmParser::parseDirectivePopSection(StringRef, SMLoc) { } bool COFFAsmParser::parseDirectiveDef(StringRef, SMLoc) { - StringRef SymbolName; + MCSymbol *Sym; - if (getParser().parseIdentifier(SymbolName)) + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - MCSymbol *Sym = getContext().getOrCreateSymbol(SymbolName); - getStreamer().beginCOFFSymbolDef(Sym); Lex(); @@ -496,8 +492,8 @@ bool COFFAsmParser::parseDirectiveEndef(StringRef, SMLoc) { } bool COFFAsmParser::parseDirectiveSecRel32(StringRef, SMLoc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return TokError("expected identifier in directive"); int64_t Offset = 0; @@ -517,8 +513,6 @@ bool COFFAsmParser::parseDirectiveSecRel32(StringRef, SMLoc) { "invalid '.secrel32' directive offset, can't be less " "than zero or greater than std::numeric_limits<uint32_t>::max()"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitCOFFSecRel32(Symbol, Offset); return false; @@ -526,8 +520,8 @@ bool COFFAsmParser::parseDirectiveSecRel32(StringRef, SMLoc) { bool COFFAsmParser::parseDirectiveRVA(StringRef, SMLoc) { auto parseOp = [&]() -> bool { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return TokError("expected identifier in directive"); int64_t Offset = 0; @@ -544,8 +538,6 @@ bool COFFAsmParser::parseDirectiveRVA(StringRef, SMLoc) { "than -2147483648 or greater than " "2147483647"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - getStreamer().emitCOFFImgRel32(Symbol, Offset); return false; }; @@ -556,75 +548,65 @@ bool COFFAsmParser::parseDirectiveRVA(StringRef, SMLoc) { } bool COFFAsmParser::parseDirectiveSafeSEH(StringRef, SMLoc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return TokError("expected identifier in directive"); if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitCOFFSafeSEH(Symbol); return false; } bool COFFAsmParser::parseDirectiveSecIdx(StringRef, SMLoc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return TokError("expected identifier in directive"); if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitCOFFSectionIndex(Symbol); return false; } bool COFFAsmParser::parseDirectiveSymIdx(StringRef, SMLoc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return TokError("expected identifier in directive"); if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitCOFFSymbolIndex(Symbol); return false; } bool COFFAsmParser::parseDirectiveSecNum(StringRef, SMLoc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return TokError("expected identifier in directive"); if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitCOFFSecNumber(Symbol); return false; } bool COFFAsmParser::parseDirectiveSecOffset(StringRef, SMLoc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return TokError("expected identifier in directive"); if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitCOFFSecOffset(Symbol); return false; @@ -679,15 +661,13 @@ bool COFFAsmParser::parseDirectiveLinkOnce(StringRef, SMLoc Loc) { } bool COFFAsmParser::parseSEHDirectiveStartProc(StringRef, SMLoc Loc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *Symbol; + if (getParser().parseSymbol(Symbol)) return true; if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitWinCFIStartProc(Symbol, Loc); return false; @@ -718,8 +698,8 @@ bool COFFAsmParser::parseSEHDirectiveEndChained(StringRef, SMLoc Loc) { } bool COFFAsmParser::parseSEHDirectiveHandler(StringRef, SMLoc Loc) { - StringRef SymbolID; - if (getParser().parseIdentifier(SymbolID)) + MCSymbol *handler; + if (getParser().parseSymbol(handler)) return true; if (getLexer().isNot(AsmToken::Comma)) @@ -736,8 +716,6 @@ bool COFFAsmParser::parseSEHDirectiveHandler(StringRef, SMLoc Loc) { if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *handler = getContext().getOrCreateSymbol(SymbolID); - Lex(); getStreamer().emitWinEHHandler(handler, unwind, except, Loc); return false; diff --git a/llvm/lib/MC/MCParser/COFFMasmParser.cpp b/llvm/lib/MC/MCParser/COFFMasmParser.cpp index 282f22fd3360..b855a765e8d2 100644 --- a/llvm/lib/MC/MCParser/COFFMasmParser.cpp +++ b/llvm/lib/MC/MCParser/COFFMasmParser.cpp @@ -444,8 +444,8 @@ bool COFFMasmParser::parseDirectiveProc(StringRef Directive, SMLoc Loc) { if (!getStreamer().getCurrentFragment()) return Error(getTok().getLoc(), "expected section directive"); - StringRef Label; - if (getParser().parseIdentifier(Label)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return Error(Loc, "expected identifier for procedure"); if (getLexer().is(AsmToken::Identifier)) { StringRef nextVal = getTok().getString(); @@ -460,11 +460,12 @@ bool COFFMasmParser::parseDirectiveProc(StringRef Directive, SMLoc Loc) { nextLoc = getTok().getLoc(); } } - MCSymbolCOFF *Sym = cast<MCSymbolCOFF>(getContext().getOrCreateSymbol(Label)); // Define symbol as simple external function - Sym->setExternal(true); - Sym->setType(COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT); + auto *COFFSym = cast<MCSymbolCOFF>(Sym); + COFFSym->setExternal(true); + COFFSym->setType(COFF::IMAGE_SYM_DTYPE_FUNCTION + << COFF::SCT_COMPLEX_TYPE_SHIFT); bool Framed = false; if (getLexer().is(AsmToken::Identifier) && @@ -475,7 +476,7 @@ bool COFFMasmParser::parseDirectiveProc(StringRef Directive, SMLoc Loc) { } getStreamer().emitLabel(Sym, Loc); - CurrentProcedures.push_back(Label); + CurrentProcedures.push_back(Sym->getName()); CurrentProceduresFramed.push_back(Framed); return false; } @@ -510,8 +511,8 @@ bool COFFMasmParser::parseDirectiveAlias(StringRef Directive, SMLoc Loc) { getParser().parseAngleBracketString(ActualName)) return Error(getTok().getLoc(), "expected <actualName>"); - MCSymbol *Alias = getContext().getOrCreateSymbol(AliasName); - MCSymbol *Actual = getContext().getOrCreateSymbol(ActualName); + MCSymbol *Alias = getContext().parseSymbol(AliasName); + MCSymbol *Actual = getContext().parseSymbol(ActualName); getStreamer().emitWeakReference(Alias, Actual); diff --git a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp index a9095b3298f5..fceb718d091c 100644 --- a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp +++ b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp @@ -501,13 +501,10 @@ bool DarwinAsmParser::parseSectionSwitch(StringRef Segment, StringRef Section, /// parseDirectiveAltEntry /// ::= .alt_entry identifier bool DarwinAsmParser::parseDirectiveAltEntry(StringRef, SMLoc) { - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - // Look up symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - if (Sym->isDefined()) return TokError(".alt_entry must preceed symbol definition"); @@ -521,13 +518,10 @@ bool DarwinAsmParser::parseDirectiveAltEntry(StringRef, SMLoc) { /// parseDirectiveDesc /// ::= .desc identifier , expression bool DarwinAsmParser::parseDirectiveDesc(StringRef, SMLoc) { - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - // Handle the identifier as the key symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - if (getLexer().isNot(AsmToken::Comma)) return TokError("unexpected token in '.desc' directive"); Lex(); @@ -560,18 +554,17 @@ bool DarwinAsmParser::parseDirectiveIndirectSymbol(StringRef, SMLoc Loc) { return Error(Loc, "indirect symbol not in a symbol pointer or stub " "section"); - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in .indirect_symbol directive"); - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - // Assembler local symbols don't make any sense here. Complain loudly. if (Sym->isTemporary()) return TokError("non-local symbol required in directive"); if (!getStreamer().emitSymbolAttribute(Sym, MCSA_IndirectSymbol)) - return TokError("unable to emit indirect symbol attribute for: " + Name); + return TokError("unable to emit indirect symbol attribute for: " + + Sym->getName()); if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in '.indirect_symbol' directive"); @@ -633,13 +626,10 @@ bool DarwinAsmParser::parseDirectiveLinkerOption(StringRef IDVal, SMLoc) { /// parseDirectiveLsym /// ::= .lsym identifier , expression bool DarwinAsmParser::parseDirectiveLsym(StringRef, SMLoc) { - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - // Handle the identifier as the key symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - if (getLexer().isNot(AsmToken::Comma)) return TokError("unexpected token in '.lsym' directive"); Lex(); @@ -826,13 +816,10 @@ bool DarwinAsmParser::parseDirectiveSubsectionsViaSymbols(StringRef, SMLoc) { /// ::= .tbss identifier, size, align bool DarwinAsmParser::parseDirectiveTBSS(StringRef, SMLoc) { SMLoc IDLoc = getLexer().getLoc(); - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - // Handle the identifier as the key symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - if (getLexer().isNot(AsmToken::Comma)) return TokError("unexpected token in directive"); Lex(); @@ -911,13 +898,10 @@ bool DarwinAsmParser::parseDirectiveZerofill(StringRef, SMLoc) { Lex(); SMLoc IDLoc = getLexer().getLoc(); - StringRef IDStr; - if (getParser().parseIdentifier(IDStr)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - // handle the identifier as the key symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(IDStr); - if (getLexer().isNot(AsmToken::Comma)) return TokError("unexpected token in directive"); Lex(); diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp index ec8b40261a6c..b04ec6451772 100644 --- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp +++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp @@ -164,7 +164,7 @@ bool ELFAsmParser::parseDirectiveSymbolAttribute(StringRef Directive, SMLoc) { continue; } - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); + MCSymbol *Sym = getContext().parseSymbol(Name); getStreamer().emitSymbolAttribute(Sym, Attr); @@ -197,10 +197,9 @@ bool ELFAsmParser::parseSectionSwitch(StringRef Section, unsigned Type, } bool ELFAsmParser::parseDirectiveSize(StringRef, SMLoc) { - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier"); - MCSymbolELF *Sym = cast<MCSymbolELF>(getContext().getOrCreateSymbol(Name)); if (getLexer().isNot(AsmToken::Comma)) return TokError("expected comma"); @@ -711,13 +710,10 @@ static MCSymbolAttr MCAttrForString(StringRef Type) { /// ::= .type identifier , %attribute /// ::= .type identifier , "attribute" bool ELFAsmParser::parseDirectiveType(StringRef, SMLoc) { - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier"); - // Handle the identifier as the key symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - bool AllowAt = getLexer().getAllowAtInIdentifier(); if (!AllowAt && !getContext().getAsmInfo()->getCommentString().starts_with("@")) @@ -789,8 +785,9 @@ bool ELFAsmParser::parseDirectiveIdent(StringRef, SMLoc) { /// parseDirectiveSymver /// ::= .symver foo, bar2@zed bool ELFAsmParser::parseDirectiveSymver(StringRef, SMLoc) { - StringRef OriginalName, Name, Action; - if (getParser().parseIdentifier(OriginalName)) + MCSymbol *OriginalSym; + StringRef Name, Action; + if (getParser().parseSymbol(OriginalSym)) return TokError("expected identifier"); if (getLexer().isNot(AsmToken::Comma)) @@ -818,8 +815,7 @@ bool ELFAsmParser::parseDirectiveSymver(StringRef, SMLoc) { } (void)parseOptionalToken(AsmToken::EndOfStatement); - getStreamer().emitELFSymverDirective( - getContext().getOrCreateSymbol(OriginalName), Name, KeepOriginalSym); + getStreamer().emitELFSymverDirective(OriginalSym, Name, KeepOriginalSym); return false; } @@ -852,8 +848,8 @@ bool ELFAsmParser::parseDirectiveVersion(StringRef, SMLoc) { bool ELFAsmParser::parseDirectiveWeakref(StringRef, SMLoc) { // FIXME: Share code with the other alias building directives. - StringRef AliasName; - if (getParser().parseIdentifier(AliasName)) + MCSymbol *Alias; + if (getParser().parseSymbol(Alias)) return TokError("expected identifier"); if (getLexer().isNot(AsmToken::Comma)) @@ -861,14 +857,10 @@ bool ELFAsmParser::parseDirectiveWeakref(StringRef, SMLoc) { Lex(); - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier"); - MCSymbol *Alias = getContext().getOrCreateSymbol(AliasName); - - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - getStreamer().emitWeakReference(Alias, Sym); return false; } diff --git a/llvm/lib/MC/MCParser/MCAsmParser.cpp b/llvm/lib/MC/MCParser/MCAsmParser.cpp index 68b9cab2492f..c1b7e57184de 100644 --- a/llvm/lib/MC/MCParser/MCAsmParser.cpp +++ b/llvm/lib/MC/MCParser/MCAsmParser.cpp @@ -163,6 +163,15 @@ bool MCAsmParser::parseGNUAttribute(SMLoc L, int64_t &Tag, return true; } +bool MCAsmParser::parseSymbol(MCSymbol *&Res) { + StringRef Name; + if (parseIdentifier(Name)) + return true; + + Res = getContext().parseSymbol(Name); + return false; +} + void MCParsedAsmOperand::dump() const { // Cannot completely remove virtual function even in release mode. #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp b/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp index 7fa05088c972..299d4b46a8a8 100644 --- a/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp +++ b/llvm/lib/MC/MCParser/MCAsmParserExtension.cpp @@ -50,8 +50,8 @@ bool MCAsmParserExtension::parseDirectiveCGProfile(StringRef, SMLoc) { if (getLexer().isNot(AsmToken::EndOfStatement)) return TokError("unexpected token in directive"); - MCSymbol *FromSym = getContext().getOrCreateSymbol(From); - MCSymbol *ToSym = getContext().getOrCreateSymbol(To); + MCSymbol *FromSym = getContext().parseSymbol(From); + MCSymbol *ToSym = getContext().parseSymbol(To); getStreamer().emitCGProfileEntry( MCSymbolRefExpr::create(FromSym, getContext(), FromLoc), diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp index f4684e64e862..3730c63b0423 100644 --- a/llvm/lib/MC/MCParser/MasmParser.cpp +++ b/llvm/lib/MC/MCParser/MasmParser.cpp @@ -1480,7 +1480,7 @@ bool MasmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc, auto VarIt = Variables.find(SymbolName.lower()); if (VarIt != Variables.end()) SymbolName = VarIt->second.Name; - Sym = getContext().getOrCreateSymbol(SymbolName); + Sym = getContext().parseSymbol(SymbolName); } // If this is an absolute variable reference, substitute it now to preserve @@ -1965,7 +1965,7 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info, if (IDVal == "@@") { Sym = Ctx.createDirectionalLocalSymbol(0); } else { - Sym = getContext().getOrCreateSymbol(IDVal); + Sym = getContext().parseSymbol(IDVal); } // End of Labels should be treated as end of line for lexing @@ -3009,8 +3009,7 @@ bool MasmParser::parseDirectiveEquate(StringRef IDVal, StringRef Name, return false; } - MCSymbol *Sym = getContext().getOrCreateSymbol(Var.Name); - + auto *Sym = getContext().parseSymbol(Var.Name); const MCConstantExpr *PrevValue = Sym->isVariable() ? dyn_cast_or_null<MCConstantExpr>(Sym->getVariableValue()) @@ -3318,7 +3317,7 @@ bool MasmParser::parseDirectiveNamedValue(StringRef TypeName, unsigned Size, StringRef Name, SMLoc NameLoc) { if (StructInProgress.empty()) { // Initialize named data value. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); + MCSymbol *Sym = getContext().parseSymbol(Name); getStreamer().emitLabel(Sym); unsigned Count; if (emitIntegralValues(Size, &Count)) @@ -3509,7 +3508,7 @@ bool MasmParser::parseDirectiveNamedRealValue(StringRef TypeName, SMLoc NameLoc) { if (StructInProgress.empty()) { // Initialize named data value. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); + MCSymbol *Sym = getContext().parseSymbol(Name); getStreamer().emitLabel(Sym); unsigned Count; if (emitRealValues(Semantics, &Count)) @@ -4003,7 +4002,7 @@ bool MasmParser::parseDirectiveNamedStructValue(const StructInfo &Structure, SMLoc DirLoc, StringRef Name) { if (StructInProgress.empty()) { // Initialize named data value. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); + MCSymbol *Sym = getContext().parseSymbol(Name); getStreamer().emitLabel(Sym); unsigned Count; if (emitStructValues(Structure, &Count)) @@ -4504,9 +4503,9 @@ bool MasmParser::parseDirectivePurgeMacro(SMLoc DirectiveLoc) { bool MasmParser::parseDirectiveExtern() { // .extern is the default - but we still need to take any provided type info. auto parseOp = [&]() -> bool { - StringRef Name; + MCSymbol *Sym; SMLoc NameLoc = getTok().getLoc(); - if (parseIdentifier(Name)) + if (parseSymbol(Sym)) return Error(NameLoc, "expected name"); if (parseToken(AsmToken::Colon)) return true; @@ -4519,10 +4518,9 @@ bool MasmParser::parseDirectiveExtern() { AsmTypeInfo Type; if (lookUpType(TypeName, Type)) return Error(TypeLoc, "unrecognized type"); - KnownType[Name.lower()] = Type; + KnownType[Sym->getName().lower()] = Type; } - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); Sym->setExternal(true); getStreamer().emitSymbolAttribute(Sym, MCSA_Extern); @@ -4538,11 +4536,10 @@ bool MasmParser::parseDirectiveExtern() { /// ::= { ".globl", ".weak", ... } [ identifier ( , identifier )* ] bool MasmParser::parseDirectiveSymbolAttribute(MCSymbolAttr Attr) { auto parseOp = [&]() -> bool { - StringRef Name; SMLoc Loc = getTok().getLoc(); - if (parseIdentifier(Name)) + MCSymbol *Sym; + if (parseSymbol(Sym)) return Error(Loc, "expected identifier"); - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); // Assembler local symbols don't make any sense here. Complain loudly. if (Sym->isTemporary()) @@ -4565,13 +4562,10 @@ bool MasmParser::parseDirectiveComm(bool IsLocal) { return true; SMLoc IDLoc = getLexer().getLoc(); - StringRef Name; - if (parseIdentifier(Name)) + MCSymbol *Sym; + if (parseSymbol(Sym)) return TokError("expected identifier in directive"); - // Handle the identifier as the key symbol. - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); - if (getLexer().isNot(AsmToken::Comma)) return TokError("unexpected token in directive"); Lex(); diff --git a/llvm/lib/MC/MCParser/WasmAsmParser.cpp b/llvm/lib/MC/MCParser/WasmAsmParser.cpp index 1f824b80bcd4..61534c6436ac 100644 --- a/llvm/lib/MC/MCParser/WasmAsmParser.cpp +++ b/llvm/lib/MC/MCParser/WasmAsmParser.cpp @@ -213,10 +213,9 @@ public: // TODO: This function is almost the same as ELFAsmParser::ParseDirectiveSize // so maybe could be shared somehow. bool parseDirectiveSize(StringRef, SMLoc Loc) { - StringRef Name; - if (Parser->parseIdentifier(Name)) + MCSymbol *Sym; + if (Parser->parseSymbol(Sym)) return TokError("expected identifier in directive"); - auto Sym = getContext().getOrCreateSymbol(Name); if (expect(AsmToken::Comma, ",")) return true; const MCExpr *Expr; @@ -241,9 +240,8 @@ public: if (!Lexer->is(AsmToken::Identifier)) return error("Expected label after .type directive, got: ", Lexer->getTok()); - auto WasmSym = cast<MCSymbolWasm>( - getStreamer().getContext().getOrCreateSymbol( - Lexer->getTok().getString())); + auto *WasmSym = cast<MCSymbolWasm>( + getStreamer().getContext().parseSymbol(Lexer->getTok().getString())); Lex(); if (!(isNext(AsmToken::Comma) && isNext(AsmToken::At) && Lexer->is(AsmToken::Identifier))) @@ -294,10 +292,9 @@ public: assert(Attr != MCSA_Invalid && "unexpected symbol attribute directive!"); if (getLexer().isNot(AsmToken::EndOfStatement)) { while (true) { - StringRef Name; - if (getParser().parseIdentifier(Name)) + MCSymbol *Sym; + if (getParser().parseSymbol(Sym)) return TokError("expected identifier in directive"); - MCSymbol *Sym = getContext().getOrCreateSymbol(Name); getStreamer().emitSymbolAttribute(Sym, Attr); if (getLexer().is(AsmToken::EndOfStatement)) break; diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp index beb472b7c7de..97f591fbf0e2 100644 --- a/llvm/lib/MC/MCSection.cpp +++ b/llvm/lib/MC/MCSection.cpp @@ -63,6 +63,8 @@ LLVM_DUMP_METHOD void MCSection::dump( raw_ostream &OS = errs(); OS << "MCSection Name:" << getName(); + if (isLinkerRelaxable()) + OS << " LinkerRelaxable"; for (auto &F : *this) { OS << '\n'; F.dump(); @@ -82,11 +84,11 @@ LLVM_DUMP_METHOD void MCSection::dump( void MCEncodedFragment::setContents(ArrayRef<char> Contents) { auto &S = getParent()->ContentStorage; - if (ContentStart + Contents.size() > ContentEnd) { + if (Contents.size() > ContentSize) { ContentStart = S.size(); S.resize_for_overwrite(S.size() + Contents.size()); } - ContentEnd = ContentStart + Contents.size(); + ContentSize = Contents.size(); llvm::copy(Contents, S.begin() + ContentStart); } diff --git a/llvm/lib/ObjCopy/COFF/COFFObject.cpp b/llvm/lib/ObjCopy/COFF/COFFObject.cpp index 5fa13391c908..91cf7e32a739 100644 --- a/llvm/lib/ObjCopy/COFF/COFFObject.cpp +++ b/llvm/lib/ObjCopy/COFF/COFFObject.cpp @@ -18,6 +18,8 @@ using namespace object; void Object::addSymbols(ArrayRef<Symbol> NewSymbols) { for (Symbol S : NewSymbols) { S.UniqueId = NextSymbolUniqueId++; + S.OriginalRawIndex = NextSymbolOriginalIndex; + NextSymbolOriginalIndex += 1 + S.Sym.NumberOfAuxSymbols; Symbols.emplace_back(S); } updateSymbols(); diff --git a/llvm/lib/ObjCopy/COFF/COFFObject.h b/llvm/lib/ObjCopy/COFF/COFFObject.h index cdd1f17fc605..6b70add1bb1b 100644 --- a/llvm/lib/ObjCopy/COFF/COFFObject.h +++ b/llvm/lib/ObjCopy/COFF/COFFObject.h @@ -89,6 +89,7 @@ struct Symbol { std::optional<size_t> WeakTargetSymbolId; size_t UniqueId; size_t RawIndex; + size_t OriginalRawIndex; bool Referenced; }; @@ -140,6 +141,7 @@ private: DenseMap<size_t, Symbol *> SymbolMap; size_t NextSymbolUniqueId = 0; + size_t NextSymbolOriginalIndex = 0; std::vector<Section> Sections; DenseMap<ssize_t, Section *> SectionMap; diff --git a/llvm/lib/ObjCopy/COFF/COFFReader.cpp b/llvm/lib/ObjCopy/COFF/COFFReader.cpp index 62a71d41ded5..9b55f76e5840 100644 --- a/llvm/lib/ObjCopy/COFF/COFFReader.cpp +++ b/llvm/lib/ObjCopy/COFF/COFFReader.cpp @@ -135,7 +135,7 @@ Error COFFReader::readSymbols(Object &Obj, bool IsBigObj) const { // it is, find the target section unique id. const coff_aux_section_definition *SD = SymRef.getSectionDefinition(); const coff_aux_weak_external *WE = SymRef.getWeakExternal(); - if (SD && SD->Selection == IMAGE_COMDAT_SELECT_ASSOCIATIVE) { + if (SD && SD->Selection == IMAGE_COMDAT_SELECT_ASSOCIATIVE && !Obj.IsPE) { int32_t Index = SD->getNumber(IsBigObj); if (Index <= 0 || static_cast<uint32_t>(Index - 1) >= Sections.size()) return createStringError(object_error::parse_failed, diff --git a/llvm/lib/ObjCopy/COFF/COFFWriter.cpp b/llvm/lib/ObjCopy/COFF/COFFWriter.cpp index 350c4aec572c..fed67d67f13a 100644 --- a/llvm/lib/ObjCopy/COFF/COFFWriter.cpp +++ b/llvm/lib/ObjCopy/COFF/COFFWriter.cpp @@ -12,6 +12,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/COFF.h" +#include "llvm/Support/CRC.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/Errc.h" #include "llvm/Support/ErrorHandling.h" #include <cstddef> @@ -92,6 +94,77 @@ Error COFFWriter::finalizeSymbolContents() { return Error::success(); } +Error COFFWriter::finalizeSymIdxContents() { + // CFGuards shouldn't be present in PE. + if (Obj.IsPE) + return Error::success(); + + // Currently handle only sections consisting only of .symidx. + // TODO: other sections such as .impcall and .hybmp$x require more complex + // handling as they have more complex layout. + auto IsSymIdxSection = [](StringRef Name) { + return Name == ".gljmp$y" || Name == ".giats$y" || Name == ".gfids$y" || + Name == ".gehcont$y"; + }; + + DenseMap<size_t, size_t> SymIdMap; + SmallDenseMap<ssize_t, coff_aux_section_definition *, 4> SecIdMap; + for (Symbol &Sym : Obj.getMutableSymbols()) { + SymIdMap[Sym.OriginalRawIndex] = Sym.RawIndex; + + // We collect only definition symbols of the sections to update the + // checksums. + if (Sym.Sym.StorageClass == IMAGE_SYM_CLASS_STATIC && + Sym.Sym.NumberOfAuxSymbols == 1 && Sym.Sym.Value == 0 && + IsSymIdxSection(Sym.Name)) + SecIdMap[Sym.TargetSectionId] = + reinterpret_cast<coff_aux_section_definition *>( + Sym.AuxData[0].Opaque); + } + + for (Section &Sec : Obj.getMutableSections()) { + if (!IsSymIdxSection(Sec.Name)) + continue; + + ArrayRef<uint8_t> RawIds = Sec.getContents(); + // Nothing to do and also the checksum will be -1 instead of 0 if we + // recalculate it on empty input. + if (RawIds.size() == 0) + continue; + + auto SecDefIt = SecIdMap.find(Sec.UniqueId); + if (SecDefIt == SecIdMap.end()) + return createStringError(object_error::invalid_symbol_index, + "section '%s' does not have the corresponding " + "symbol or the symbol has unexpected format", + Sec.Name.str().c_str()); + + // Create updated content. + ArrayRef<support::ulittle32_t> Ids( + reinterpret_cast<const support::ulittle32_t *>(RawIds.data()), + RawIds.size() / 4); + std::vector<support::ulittle32_t> NewIds; + for (support::ulittle32_t Id : Ids) { + auto SymIdIt = SymIdMap.find(Id); + if (SymIdIt == SymIdMap.end()) + return createStringError(object_error::invalid_symbol_index, + "section '%s' contains a .symidx (%d) that is " + "incorrect or was stripped", + Sec.Name.str().c_str(), Id.value()); + NewIds.push_back(support::ulittle32_t(SymIdIt->getSecond())); + } + ArrayRef<uint8_t> NewRawIds(reinterpret_cast<uint8_t *>(NewIds.data()), + RawIds.size()); + // Update the checksum. + JamCRC JC(/*Init=*/0); + JC.update(NewRawIds); + SecDefIt->getSecond()->CheckSum = JC.getCRC(); + // Set new content. + Sec.setOwnedContents(NewRawIds.vec()); + } + return Error::success(); +} + void COFFWriter::layoutSections() { for (auto &S : Obj.getMutableSections()) { if (S.Header.SizeOfRawData > 0) @@ -183,6 +256,8 @@ Error COFFWriter::finalize(bool IsBigObj) { return E; if (Error E = finalizeSymbolContents()) return E; + if (Error E = finalizeSymIdxContents()) + return E; size_t SizeOfHeaders = 0; FileAlignment = 1; diff --git a/llvm/lib/ObjCopy/COFF/COFFWriter.h b/llvm/lib/ObjCopy/COFF/COFFWriter.h index b7dca69e9a81..66d7f01c87f1 100644 --- a/llvm/lib/ObjCopy/COFF/COFFWriter.h +++ b/llvm/lib/ObjCopy/COFF/COFFWriter.h @@ -34,6 +34,7 @@ class COFFWriter { template <class SymbolTy> std::pair<size_t, size_t> finalizeSymbolTable(); Error finalizeRelocTargets(); Error finalizeSymbolContents(); + Error finalizeSymIdxContents(); void layoutSections(); Expected<size_t> finalizeStringTable(); diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp index 0e13d32bbe52..a6b56ae77cf2 100644 --- a/llvm/lib/Object/ELFObjectFile.cpp +++ b/llvm/lib/Object/ELFObjectFile.cpp @@ -620,7 +620,10 @@ StringRef ELFObjectFileBase::getAMDGPUCPUName() const { StringRef ELFObjectFileBase::getNVPTXCPUName() const { assert(getEMachine() == ELF::EM_CUDA); - unsigned SM = getPlatformFlags() & ELF::EF_CUDA_SM; + unsigned SM = getEIdentABIVersion() == ELF::ELFABIVERSION_CUDA_V1 + ? getPlatformFlags() & ELF::EF_CUDA_SM + : (getPlatformFlags() & ELF::EF_CUDA_SM_MASK) >> + ELF::EF_CUDA_SM_OFFSET; switch (SM) { // Fermi architecture. @@ -672,6 +675,8 @@ StringRef ELFObjectFileBase::getNVPTXCPUName() const { return "sm_86"; case ELF::EF_CUDA_SM87: return "sm_87"; + case ELF::EF_CUDA_SM88: + return "sm_88"; // Ada architecture. case ELF::EF_CUDA_SM89: @@ -679,7 +684,30 @@ StringRef ELFObjectFileBase::getNVPTXCPUName() const { // Hopper architecture. case ELF::EF_CUDA_SM90: - return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS ? "sm_90a" : "sm_90"; + return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS_V1 ? "sm_90a" + : "sm_90"; + + // Blackwell architecture. + case ELF::EF_CUDA_SM100: + return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS ? "sm_100a" + : "sm_100"; + case ELF::EF_CUDA_SM101: + return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS ? "sm_101a" + : "sm_101"; + case ELF::EF_CUDA_SM103: + return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS ? "sm_103a" + : "sm_103"; + case ELF::EF_CUDA_SM110: + return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS ? "sm_110a" + : "sm_110"; + + // Blackwell architecture. + case ELF::EF_CUDA_SM120: + return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS ? "sm_120a" + : "sm_120"; + case ELF::EF_CUDA_SM121: + return getPlatformFlags() & ELF::EF_CUDA_ACCELERATORS ? "sm_121a" + : "sm_121"; default: llvm_unreachable("Unknown EF_CUDA_SM value"); } diff --git a/llvm/lib/Support/BLAKE3/blake3_dispatch.c b/llvm/lib/Support/BLAKE3/blake3_dispatch.c index d00580fe3519..19918aa708b2 100644 --- a/llvm/lib/Support/BLAKE3/blake3_dispatch.c +++ b/llvm/lib/Support/BLAKE3/blake3_dispatch.c @@ -236,7 +236,7 @@ void blake3_xof_many(const uint32_t cv[8], #if defined(IS_X86) const enum cpu_feature features = get_cpu_features(); MAYBE_UNUSED(features); -#if !defined(_WIN32) && !defined(BLAKE3_NO_AVX512) +#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) if (features & AVX512VL) { blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); return; diff --git a/llvm/lib/Support/BLAKE3/blake3_impl.h b/llvm/lib/Support/BLAKE3/blake3_impl.h index deed079e468a..dd71e729f208 100644 --- a/llvm/lib/Support/BLAKE3/blake3_impl.h +++ b/llvm/lib/Support/BLAKE3/blake3_impl.h @@ -324,7 +324,7 @@ void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); -#if !defined(_WIN32) +#if !defined(_WIN32) && !defined(__CYGWIN__) LLVM_LIBRARY_VISIBILITY void blake3_xof_many_avx512(const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], diff --git a/llvm/lib/Support/BLAKE3/blake3_neon.c b/llvm/lib/Support/BLAKE3/blake3_neon.c index 9629e1083686..ee36721f8757 100644 --- a/llvm/lib/Support/BLAKE3/blake3_neon.c +++ b/llvm/lib/Support/BLAKE3/blake3_neon.c @@ -245,10 +245,11 @@ INLINE void load_counters4(uint64_t counter, bool increment_counter, counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); } -void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { +static void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { uint32x4_t h_vecs[8] = { set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), diff --git a/llvm/lib/Support/BLAKE3/llvm_blake3_prefix.h b/llvm/lib/Support/BLAKE3/llvm_blake3_prefix.h index d5be360815ad..d24657465dd8 100644 --- a/llvm/lib/Support/BLAKE3/llvm_blake3_prefix.h +++ b/llvm/lib/Support/BLAKE3/llvm_blake3_prefix.h @@ -10,7 +10,9 @@ #define blake3_hasher llvm_blake3_hasher #define blake3_chunk_state llvm_blake3_chunk_state #define blake3_compress_in_place llvm_blake3_compress_in_place +#define blake3_compress_subtree_wide llvm_blake3_compress_subtree_wide #define blake3_compress_xof llvm_blake3_compress_xof +#define blake3_xof_many llvm_blake3_xof_many #define blake3_hash_many llvm_blake3_hash_many #define blake3_simd_degree llvm_blake3_simd_degree #define blake3_compress_in_place_portable llvm_blake3_compress_in_place_portable diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp index 601f11f6d23c..1c4645ad8364 100644 --- a/llvm/lib/Support/MemoryBuffer.cpp +++ b/llvm/lib/Support/MemoryBuffer.cpp @@ -501,8 +501,14 @@ getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize, std::unique_ptr<MB> Result( new (NamedBufferAlloc(Filename)) MemoryBufferMMapFile<MB>( RequiresNullTerminator, FD, MapSize, Offset, EC)); - if (!EC) - return std::move(Result); + if (!EC) { + // On at least Linux, and possibly on other systems, mmap may return pages + // from the page cache that are not properly filled with trailing zeroes, + // if some prior user of the page wrote non-zero bytes. Detect this and + // don't use mmap in that case. + if (!RequiresNullTerminator || *Result->getBufferEnd() == '\0') + return std::move(Result); + } } #ifdef __MVS__ diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index c4b43e1b5126..4bd650b37792 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -2195,13 +2195,24 @@ void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) { if (BrTarget == AddrDisc) report_fatal_error("Branch target is signed with its own value"); - // If we are printing BLRA pseudo instruction, then x16 and x17 are - // implicit-def'ed by the MI and AddrDisc is not used as any other input, so - // try to save one MOV by setting MayUseAddrAsScratch. + // If we are printing BLRA pseudo, try to save one MOV by making use of the + // fact that x16 and x17 are described as clobbered by the MI instruction and + // AddrDisc is not used as any other input. + // + // Back in the day, emitPtrauthDiscriminator was restricted to only returning + // either x16 or x17, meaning the returned register is always among the + // implicit-def'ed registers of BLRA pseudo. Now this property can be violated + // if isX16X17Safer predicate is false, thus manually check if AddrDisc is + // among x16 and x17 to prevent clobbering unexpected registers. + // // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not // declared as clobbering x16/x17. + // + // FIXME: Make use of `killed` flags and register masks instead. + bool AddrDiscIsImplicitDef = + IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17); Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17, - /*MayUseAddrAsScratch=*/IsCall); + AddrDiscIsImplicitDef); bool IsZeroDisc = DiscReg == AArch64::XZR; unsigned Opc; @@ -2930,8 +2941,15 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { // See the comments in emitPtrauthBranch. if (Callee == AddrDisc) report_fatal_error("Call target is signed with its own value"); + + // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is + // no longer restricted to only reusing AddrDisc when it is X16 or X17 + // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this + // restriction manually not to clobber an unexpected register. + bool AddrDiscIsImplicitDef = + AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17; Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg, - /*MayUseAddrAsScratch=*/true); + AddrDiscIsImplicitDef); const bool IsZero = DiscReg == AArch64::XZR; const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ}, diff --git a/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp b/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp index 3436dc9ef452..137ff898e86a 100644 --- a/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp +++ b/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp @@ -30,6 +30,14 @@ using namespace llvm; #define AARCH64_BRANCH_TARGETS_NAME "AArch64 Branch Targets" namespace { +// BTI HINT encoding: base (32) plus 'c' (2) and/or 'j' (4). +enum : unsigned { + BTIBase = 32, // Base immediate for BTI HINT + BTIC = 1u << 1, // 2 + BTIJ = 1u << 2, // 4 + BTIMask = BTIC | BTIJ, +}; + class AArch64BranchTargets : public MachineFunctionPass { public: static char ID; @@ -42,6 +50,7 @@ private: void addBTI(MachineBasicBlock &MBB, bool CouldCall, bool CouldJump, bool NeedsWinCFI); }; + } // end anonymous namespace char AArch64BranchTargets::ID = 0; @@ -62,9 +71,8 @@ bool AArch64BranchTargets::runOnMachineFunction(MachineFunction &MF) { if (!MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement()) return false; - LLVM_DEBUG( - dbgs() << "********** AArch64 Branch Targets **********\n" - << "********** Function: " << MF.getName() << '\n'); + LLVM_DEBUG(dbgs() << "********** AArch64 Branch Targets **********\n" + << "********** Function: " << MF.getName() << '\n'); const Function &F = MF.getFunction(); // LLVM does not consider basic blocks which are the targets of jump tables @@ -103,6 +111,12 @@ bool AArch64BranchTargets::runOnMachineFunction(MachineFunction &MF) { JumpTableTargets.count(&MBB)) CouldJump = true; + if (MBB.isEHPad()) { + if (HasWinCFI && (MBB.isEHFuncletEntry() || MBB.isCleanupFuncletEntry())) + CouldCall = true; + else + CouldJump = true; + } if (CouldCall || CouldJump) { addBTI(MBB, CouldCall, CouldJump, HasWinCFI); MadeChange = true; @@ -130,7 +144,12 @@ void AArch64BranchTargets::addBTI(MachineBasicBlock &MBB, bool CouldCall, auto MBBI = MBB.begin(); - // Skip the meta instructions, those will be removed anyway. + // If the block starts with EH_LABEL(s), skip them first. + while (MBBI != MBB.end() && MBBI->isEHLabel()) { + ++MBBI; + } + + // Skip meta/CFI/etc. (and EMITBKEY) to reach the first executable insn. for (; MBBI != MBB.end() && (MBBI->isMetaInstruction() || MBBI->getOpcode() == AArch64::EMITBKEY); ++MBBI) @@ -138,16 +157,21 @@ void AArch64BranchTargets::addBTI(MachineBasicBlock &MBB, bool CouldCall, // SCTLR_EL1.BT[01] is set to 0 by default which means // PACI[AB]SP are implicitly BTI C so no BTI C instruction is needed there. - if (MBBI != MBB.end() && HintNum == 34 && + if (MBBI != MBB.end() && ((HintNum & BTIMask) == BTIC) && (MBBI->getOpcode() == AArch64::PACIASP || MBBI->getOpcode() == AArch64::PACIBSP)) return; - if (HasWinCFI && MBBI->getFlag(MachineInstr::FrameSetup)) { - BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()), - TII->get(AArch64::SEH_Nop)); + // Insert BTI exactly at the first executable instruction. + const DebugLoc DL = MBB.findDebugLoc(MBBI); + MachineInstr *BTI = BuildMI(MBB, MBBI, DL, TII->get(AArch64::HINT)) + .addImm(HintNum) + .getInstr(); + + // WinEH: put .seh_nop after BTI when the first real insn is FrameSetup. + if (HasWinCFI && MBBI != MBB.end() && + MBBI->getFlag(MachineInstr::FrameSetup)) { + auto AfterBTI = std::next(MachineBasicBlock::iterator(BTI)); + BuildMI(MBB, AfterBTI, DL, TII->get(AArch64::SEH_Nop)); } - BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()), - TII->get(AArch64::HINT)) - .addImm(HintNum); } diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td index 9973df865ea1..12159a951973 100644 --- a/llvm/lib/Target/AArch64/AArch64Features.td +++ b/llvm/lib/Target/AArch64/AArch64Features.td @@ -923,8 +923,8 @@ def HasV9_5aOps : Architecture64<9, 5, "a", "v9.5a", [HasV9_4aOps, FeatureCPA], !listconcat(HasV9_4aOps.DefaultExts, [FeatureCPA, FeatureLUT, FeatureFAMINMAX])>; def HasV9_6aOps : Architecture64<9, 6, "a", "v9.6a", - [HasV9_5aOps, FeatureCMPBR, FeatureFPRCVT, FeatureSVE2p2, FeatureLSUI, FeatureOCCMO], - !listconcat(HasV9_5aOps.DefaultExts, [FeatureCMPBR, FeatureFPRCVT, FeatureSVE2p2, + [HasV9_5aOps, FeatureCMPBR, FeatureLSUI, FeatureOCCMO], + !listconcat(HasV9_5aOps.DefaultExts, [FeatureCMPBR, FeatureLSUI, FeatureOCCMO])>; def HasV8_0rOps : Architecture64<8, 0, "r", "v8r", [ //v8.1 diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index ddc685fae5e9..bbcd6824473c 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -2148,7 +2148,7 @@ let Predicates = [HasPAuth] in { i64imm:$Disc, GPR64:$AddrDisc), [], "$AuthVal = $Val">, Sched<[WriteI, ReadI]> { let isCodeGenOnly = 1; - let hasSideEffects = 0; + let hasSideEffects = 1; let mayStore = 0; let mayLoad = 0; let Size = 32; diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 07baf29ce701..193be62b28ad 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -3724,7 +3724,7 @@ InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode, InstructionCost AArch64TTIImpl::getVectorInstrCostHelper( unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, - bool HasRealUse, const Instruction *I, Value *Scalar, + const Instruction *I, Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const { assert(Val->isVectorTy() && "This must be a vector type"); @@ -3744,12 +3744,10 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper( } // The element at index zero is already inside the vector. - // - For a physical (HasRealUse==true) insert-element or extract-element + // - For a insert-element or extract-element // instruction that extracts integers, an explicit FPR -> GPR move is // needed. So it has non-zero cost. - // - For the rest of cases (virtual instruction or element type is float), - // consider the instruction free. - if (Index == 0 && (!HasRealUse || !Val->getScalarType()->isIntegerTy())) + if (Index == 0 && !Val->getScalarType()->isIntegerTy()) return 0; // This is recognising a LD1 single-element structure to one lane of one @@ -3899,25 +3897,28 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index, const Value *Op0, const Value *Op1) const { - bool HasRealUse = - Opcode == Instruction::InsertElement && Op0 && !isa<UndefValue>(Op0); - return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, HasRealUse); + // Treat insert at lane 0 into a poison vector as having zero cost. This + // ensures vector broadcasts via an insert + shuffle (and will be lowered to a + // single dup) are treated as cheap. + if (Opcode == Instruction::InsertElement && Index == 0 && Op0 && + isa<PoisonValue>(Op0)) + return 0; + return getVectorInstrCostHelper(Opcode, Val, CostKind, Index); } InstructionCost AArch64TTIImpl::getVectorInstrCost( unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const { - return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, false, nullptr, - Scalar, ScalarUserAndIdx); + return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, nullptr, Scalar, + ScalarUserAndIdx); } InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I, Type *Val, TTI::TargetCostKind CostKind, unsigned Index) const { - return getVectorInstrCostHelper(I.getOpcode(), Val, CostKind, Index, - true /* HasRealUse */, &I); + return getVectorInstrCostHelper(I.getOpcode(), Val, CostKind, Index, &I); } InstructionCost AArch64TTIImpl::getScalarizationOverhead( diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index ff0ab68a16a8..b27eb2ef7a39 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -65,16 +65,14 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> { // A helper function called by 'getVectorInstrCost'. // - // 'Val' and 'Index' are forwarded from 'getVectorInstrCost'; 'HasRealUse' - // indicates whether the vector instruction is available in the input IR or - // just imaginary in vectorizer passes. - /// \param ScalarUserAndIdx encodes the information about extracts from a + // 'Val' and 'Index' are forwarded from 'getVectorInstrCost'; + // \param ScalarUserAndIdx encodes the information about extracts from a /// vector with 'Scalar' being the value being extracted,'User' being the user /// of the extract(nullptr if user is not known before vectorization) and /// 'Idx' being the extract lane. InstructionCost getVectorInstrCostHelper( unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, - bool HasRealUse, const Instruction *I = nullptr, Value *Scalar = nullptr, + const Instruction *I = nullptr, Value *Scalar = nullptr, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx = {}) const; public: diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 3414fe758eff..7b93382d1281 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -392,8 +392,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, // Library functions. These default to Expand, but we have instructions // for them. setOperationAction({ISD::FCEIL, ISD::FPOW, ISD::FABS, ISD::FFLOOR, - ISD::FROUNDEVEN, ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM}, - MVT::f32, Legal); + ISD::FROUNDEVEN, ISD::FTRUNC}, + {MVT::f16, MVT::f32}, Legal); + setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, MVT::f32, Legal); setOperationAction(ISD::FLOG2, MVT::f32, Custom); setOperationAction(ISD::FROUND, {MVT::f32, MVT::f64}, Custom); @@ -413,9 +414,10 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, setOperationAction(ISD::FREM, {MVT::f16, MVT::f32, MVT::f64}, Custom); - if (Subtarget->has16BitInsts()) + if (Subtarget->has16BitInsts()) { setOperationAction(ISD::IS_FPCLASS, {MVT::f16, MVT::f32, MVT::f64}, Legal); - else { + setOperationAction({ISD::FLOG2, ISD::FEXP2}, MVT::f16, Legal); + } else { setOperationAction(ISD::IS_FPCLASS, {MVT::f32, MVT::f64}, Legal); setOperationAction({ISD::FLOG2, ISD::FEXP2}, MVT::f16, Custom); } diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index fb72bab03e75..9593038ff2c9 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -370,6 +370,11 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::FMINNUM, VT, Legal); setOperationAction(ISD::FMAXNUM, VT, Legal); setOperationAction(ISD::FROUND, VT, Legal); + setOperationAction(ISD::FROUNDEVEN, VT, Legal); + setOperationAction(ISD::FRINT, VT, Legal); + setOperationAction(ISD::FTRUNC, VT, Legal); + setOperationAction(ISD::FFLOOR, VT, Legal); + setOperationAction(ISD::FCEIL, VT, Legal); setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom); setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); @@ -1507,6 +1512,12 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_, setOperationAction(ISD::FLOG2, MVT::f16, Promote); setOperationAction(ISD::FROUND, MVT::f16, Legal); + setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal); + setOperationAction(ISD::FTRUNC, MVT::f16, Legal); + setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal); + setOperationAction(ISD::FRINT, MVT::f16, Legal); + setOperationAction(ISD::FFLOOR, MVT::f16, Legal); + setOperationAction(ISD::FCEIL, MVT::f16, Legal); } if (Subtarget->hasNEON()) { diff --git a/llvm/lib/Target/AVR/AVRISelLowering.cpp b/llvm/lib/Target/AVR/AVRISelLowering.cpp index 3955f2a252e7..25ad9eccbce5 100644 --- a/llvm/lib/Target/AVR/AVRISelLowering.cpp +++ b/llvm/lib/Target/AVR/AVRISelLowering.cpp @@ -669,7 +669,7 @@ SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, default: { // Turn lhs < rhs with lhs constant into rhs >= lhs+1, this allows // us to fold the constant into the cmp instruction. - RHS = DAG.getConstant(C->getSExtValue() + 1, DL, VT); + RHS = DAG.getSignedConstant(C->getSExtValue() + 1, DL, VT); CC = ISD::SETGE; break; } @@ -713,7 +713,10 @@ SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, // Turn lhs < rhs with lhs constant into rhs >= lhs+1, this allows us to // fold the constant into the cmp instruction. if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { - RHS = DAG.getConstant(C->getSExtValue() + 1, DL, VT); + // Doing a "icmp ugt i16 65535, %0" comparison should have been converted + // already to something else. Assert to make sure this assumption holds. + assert((!C->isAllOnes()) && "integer overflow in comparison transform"); + RHS = DAG.getConstant(C->getZExtValue() + 1, DL, VT); CC = ISD::SETUGE; break; } diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp index 1e29a0f1e85a..9bf14ab59c7c 100644 --- a/llvm/lib/Target/BPF/BTFDebug.cpp +++ b/llvm/lib/Target/BPF/BTFDebug.cpp @@ -14,6 +14,7 @@ #include "BPF.h" #include "BPFCORE.h" #include "MCTargetDesc/BPFMCTargetDesc.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -23,6 +24,7 @@ #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCStreamer.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Target/TargetLoweringObjectFile.h" @@ -93,7 +95,24 @@ void BTFTypeDerived::completeType(BTFDebug &BDebug) { return; IsCompleted = true; - BTFType.NameOff = BDebug.addString(Name); + switch (Kind) { + case BTF::BTF_KIND_PTR: + case BTF::BTF_KIND_CONST: + case BTF::BTF_KIND_VOLATILE: + case BTF::BTF_KIND_RESTRICT: + // Debug info might contain names for these types, but given that we want + // to keep BTF minimal and naming reference types doesn't bring any value + // (what matters is the completeness of the base type), we don't emit them. + // + // Furthermore, the Linux kernel refuses to load BPF programs that contain + // BTF with these types named: + // https://elixir.bootlin.com/linux/v6.17.1/source/kernel/bpf/btf.c#L2586 + BTFType.NameOff = 0; + break; + default: + BTFType.NameOff = BDebug.addString(Name); + break; + } if (NeedsFixup || !DTy) return; @@ -301,21 +320,59 @@ void BTFTypeStruct::completeType(BTFDebug &BDebug) { BTFType.NameOff = BDebug.addString(STy->getName()); + if (STy->getTag() == dwarf::DW_TAG_variant_part) { + // Variant parts might have a discriminator, which has its own memory + // location, and variants, which share the memory location afterwards. LLVM + // DI doesn't consider discriminator as an element and instead keeps + // it as a separate reference. + // To keep BTF simple, let's represent the structure as an union with + // discriminator as the first element. + // The offsets inside variant types are already handled correctly in the + // DI. + const auto *DTy = STy->getDiscriminator(); + if (DTy) { + struct BTF::BTFMember Discriminator; + + Discriminator.NameOff = BDebug.addString(DTy->getName()); + Discriminator.Offset = DTy->getOffsetInBits(); + const auto *BaseTy = DTy->getBaseType(); + Discriminator.Type = BDebug.getTypeId(BaseTy); + + Members.push_back(Discriminator); + } + } + // Add struct/union members. const DINodeArray Elements = STy->getElements(); for (const auto *Element : Elements) { struct BTF::BTFMember BTFMember; - const auto *DDTy = cast<DIDerivedType>(Element); - BTFMember.NameOff = BDebug.addString(DDTy->getName()); - if (HasBitField) { - uint8_t BitFieldSize = DDTy->isBitField() ? DDTy->getSizeInBits() : 0; - BTFMember.Offset = BitFieldSize << 24 | DDTy->getOffsetInBits(); - } else { - BTFMember.Offset = DDTy->getOffsetInBits(); + switch (Element->getTag()) { + case dwarf::DW_TAG_member: { + const auto *DDTy = cast<DIDerivedType>(Element); + + BTFMember.NameOff = BDebug.addString(DDTy->getName()); + if (HasBitField) { + uint8_t BitFieldSize = DDTy->isBitField() ? DDTy->getSizeInBits() : 0; + BTFMember.Offset = BitFieldSize << 24 | DDTy->getOffsetInBits(); + } else { + BTFMember.Offset = DDTy->getOffsetInBits(); + } + const auto *BaseTy = tryRemoveAtomicType(DDTy->getBaseType()); + BTFMember.Type = BDebug.getTypeId(BaseTy); + break; + } + case dwarf::DW_TAG_variant_part: { + const auto *DCTy = dyn_cast<DICompositeType>(Element); + + BTFMember.NameOff = BDebug.addString(DCTy->getName()); + BTFMember.Offset = DCTy->getOffsetInBits(); + BTFMember.Type = BDebug.getTypeId(DCTy); + break; + } + default: + llvm_unreachable("Unexpected DI tag of a struct/union element"); } - const auto *BaseTy = tryRemoveAtomicType(DDTy->getBaseType()); - BTFMember.Type = BDebug.getTypeId(BaseTy); Members.push_back(BTFMember); } } @@ -672,16 +729,28 @@ void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct, uint32_t &TypeId) { const DINodeArray Elements = CTy->getElements(); uint32_t VLen = Elements.size(); + // Variant parts might have a discriminator. LLVM DI doesn't consider it as + // an element and instead keeps it as a separate reference. But we represent + // it as an element in BTF. + if (CTy->getTag() == dwarf::DW_TAG_variant_part) { + const auto *DTy = CTy->getDiscriminator(); + if (DTy) { + visitTypeEntry(DTy); + VLen++; + } + } if (VLen > BTF::MAX_VLEN) return; // Check whether we have any bitfield members or not bool HasBitField = false; for (const auto *Element : Elements) { - auto E = cast<DIDerivedType>(Element); - if (E->isBitField()) { - HasBitField = true; - break; + if (Element->getTag() == dwarf::DW_TAG_member) { + auto E = cast<DIDerivedType>(Element); + if (E->isBitField()) { + HasBitField = true; + break; + } } } @@ -696,9 +765,22 @@ void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct, // Visit all struct members. int FieldNo = 0; for (const auto *Element : Elements) { - const auto Elem = cast<DIDerivedType>(Element); - visitTypeEntry(Elem); - processDeclAnnotations(Elem->getAnnotations(), TypeId, FieldNo); + switch (Element->getTag()) { + case dwarf::DW_TAG_member: { + const auto Elem = cast<DIDerivedType>(Element); + visitTypeEntry(Elem); + processDeclAnnotations(Elem->getAnnotations(), TypeId, FieldNo); + break; + } + case dwarf::DW_TAG_variant_part: { + const auto Elem = cast<DICompositeType>(Element); + visitTypeEntry(Elem); + processDeclAnnotations(Elem->getAnnotations(), TypeId, FieldNo); + break; + } + default: + llvm_unreachable("Unexpected DI tag of a struct/union element"); + } FieldNo++; } } @@ -781,16 +863,25 @@ void BTFDebug::visitFwdDeclType(const DICompositeType *CTy, bool IsUnion, void BTFDebug::visitCompositeType(const DICompositeType *CTy, uint32_t &TypeId) { auto Tag = CTy->getTag(); - if (Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type) { + switch (Tag) { + case dwarf::DW_TAG_structure_type: + case dwarf::DW_TAG_union_type: + case dwarf::DW_TAG_variant_part: // Handle forward declaration differently as it does not have members. if (CTy->isForwardDecl()) visitFwdDeclType(CTy, Tag == dwarf::DW_TAG_union_type, TypeId); else visitStructType(CTy, Tag == dwarf::DW_TAG_structure_type, TypeId); - } else if (Tag == dwarf::DW_TAG_array_type) + break; + case dwarf::DW_TAG_array_type: visitArrayType(CTy, TypeId); - else if (Tag == dwarf::DW_TAG_enumeration_type) + break; + case dwarf::DW_TAG_enumeration_type: visitEnumType(CTy, TypeId); + break; + default: + llvm_unreachable("Unexpected DI tag of a composite type"); + } } bool BTFDebug::IsForwardDeclCandidate(const DIType *Base) { diff --git a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp index 5bd31707acb6..c48cf5e6353a 100644 --- a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp +++ b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp @@ -43,12 +43,12 @@ namespace { class HexagonDisassembler : public MCDisassembler { public: std::unique_ptr<MCInstrInfo const> const MCII; - std::unique_ptr<MCInst *> CurrentBundle; + mutable std::unique_ptr<MCInst> CurrentBundle; mutable MCInst const *CurrentExtender; HexagonDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx, MCInstrInfo const *MCII) - : MCDisassembler(STI, Ctx), MCII(MCII), CurrentBundle(new MCInst *), + : MCDisassembler(STI, Ctx), MCII(MCII), CurrentBundle(nullptr), CurrentExtender(nullptr) {} DecodeStatus getSingleInstruction(MCInst &Instr, MCInst &MCB, @@ -57,7 +57,27 @@ public: DecodeStatus getInstruction(MCInst &Instr, uint64_t &Size, ArrayRef<uint8_t> Bytes, uint64_t Address, raw_ostream &CStream) const override; + + DecodeStatus getInstructionBundle(MCInst &Instr, uint64_t &Size, + ArrayRef<uint8_t> Bytes, uint64_t Address, + raw_ostream &CStream) const override; + void remapInstruction(MCInst &Instr) const; + + Expected<bool> onSymbolStart(SymbolInfoTy &Symbol, uint64_t &Size, + ArrayRef<uint8_t> Bytes, + uint64_t Address) const override; + +private: + bool makeBundle(ArrayRef<uint8_t> Bytes, uint64_t Address, + uint64_t &BytesToSkip, raw_ostream &CS) const; + + void resetBundle() const { + CurrentBundle.reset(); + CurrentInstruction = nullptr; + } + + mutable MCOperand *CurrentInstruction = nullptr; }; static uint64_t fullValue(HexagonDisassembler const &Disassembler, MCInst &MI, @@ -171,43 +191,88 @@ LLVMInitializeHexagonDisassembler() { createHexagonDisassembler); } -DecodeStatus HexagonDisassembler::getInstruction(MCInst &MI, uint64_t &Size, - ArrayRef<uint8_t> Bytes, - uint64_t Address, - raw_ostream &CS) const { - CommentStream = &CS; - - DecodeStatus Result = DecodeStatus::Success; +bool HexagonDisassembler::makeBundle(ArrayRef<uint8_t> Bytes, uint64_t Address, + uint64_t &BytesToSkip, + raw_ostream &CS) const { bool Complete = false; - Size = 0; + DecodeStatus Result = DecodeStatus::Success; - *CurrentBundle = &MI; - MI.setOpcode(Hexagon::BUNDLE); - MI.addOperand(MCOperand::createImm(0)); + CurrentBundle.reset(new MCInst); + CurrentBundle->setOpcode(Hexagon::BUNDLE); + CurrentBundle->addOperand(MCOperand::createImm(0)); while (Result == Success && !Complete) { if (Bytes.size() < HEXAGON_INSTR_SIZE) - return MCDisassembler::Fail; + return false; MCInst *Inst = getContext().createMCInst(); - Result = getSingleInstruction(*Inst, MI, Bytes, Address, CS, Complete); - MI.addOperand(MCOperand::createInst(Inst)); - Size += HEXAGON_INSTR_SIZE; + Result = getSingleInstruction(*Inst, *CurrentBundle, Bytes, Address, CS, + Complete); + CurrentBundle->addOperand(MCOperand::createInst(Inst)); + BytesToSkip += HEXAGON_INSTR_SIZE; Bytes = Bytes.slice(HEXAGON_INSTR_SIZE); } if (Result == MCDisassembler::Fail) - return Result; - if (Size > HEXAGON_MAX_PACKET_SIZE) - return MCDisassembler::Fail; + return false; + if (BytesToSkip > HEXAGON_MAX_PACKET_SIZE) + return false; const auto ArchSTI = Hexagon_MC::getArchSubtarget(&STI); const auto STI_ = (ArchSTI != nullptr) ? *ArchSTI : STI; - HexagonMCChecker Checker(getContext(), *MCII, STI_, MI, + HexagonMCChecker Checker(getContext(), *MCII, STI_, *CurrentBundle, *getContext().getRegisterInfo(), false); if (!Checker.check()) - return MCDisassembler::Fail; - remapInstruction(MI); + return false; + remapInstruction(*CurrentBundle); + return true; +} + +DecodeStatus HexagonDisassembler::getInstruction(MCInst &MI, uint64_t &Size, + ArrayRef<uint8_t> Bytes, + uint64_t Address, + raw_ostream &CS) const { + CommentStream = &CS; + + Size = 0; + uint64_t BytesToSkip = 0; + + if (!CurrentBundle) { + if (!makeBundle(Bytes, Address, BytesToSkip, CS)) { + Size = BytesToSkip; + resetBundle(); + return MCDisassembler::Fail; + } + CurrentInstruction = (CurrentBundle->begin() + 1); + } + + MI = *(CurrentInstruction->getInst()); + Size = HEXAGON_INSTR_SIZE; + if (++CurrentInstruction == CurrentBundle->end()) + resetBundle(); return MCDisassembler::Success; } +DecodeStatus HexagonDisassembler::getInstructionBundle(MCInst &MI, + uint64_t &Size, + ArrayRef<uint8_t> Bytes, + uint64_t Address, + raw_ostream &CS) const { + CommentStream = &CS; + Size = 0; + uint64_t BytesToSkip = 0; + assert(!CurrentBundle); + + if (!makeBundle(Bytes, Address, BytesToSkip, CS)) { + Size = BytesToSkip; + resetBundle(); + return MCDisassembler::Fail; + } + + MI = *CurrentBundle; + Size = HEXAGON_INSTR_SIZE * HexagonMCInstrInfo::bundleSize(MI); + resetBundle(); + + return Success; +} + void HexagonDisassembler::remapInstruction(MCInst &Instr) const { for (auto I: HexagonMCInstrInfo::bundleInstructions(Instr)) { auto &MI = const_cast<MCInst &>(*I.getInst()); @@ -465,6 +530,9 @@ DecodeStatus HexagonDisassembler::getSingleInstruction(MCInst &MI, MCInst &MCB, MI.insert(MI.begin() + 1, MCOperand::createExpr(MCConstantExpr::create(-1, getContext()))); break; + case Hexagon::Y4_crswap10: + MI.addOperand(MCOperand::createReg(Hexagon::SGP1_0)); + break; default: break; } @@ -482,7 +550,7 @@ DecodeStatus HexagonDisassembler::getSingleInstruction(MCInst &MI, MCInst &MCB, unsigned Offset = 1; bool Vector = HexagonMCInstrInfo::isVector(*MCII, MI); bool PrevVector = false; - auto Instructions = HexagonMCInstrInfo::bundleInstructions(**CurrentBundle); + auto Instructions = HexagonMCInstrInfo::bundleInstructions(*CurrentBundle); auto i = Instructions.end() - 1; for (auto n = Instructions.begin() - 1;; --i, ++Offset) { if (i == n) @@ -540,6 +608,18 @@ DecodeStatus HexagonDisassembler::getSingleInstruction(MCInst &MI, MCInst &MCB, return Result; } +Expected<bool> HexagonDisassembler::onSymbolStart(SymbolInfoTy &Symbol, + uint64_t &Size, + ArrayRef<uint8_t> Bytes, + uint64_t Address) const { + // At the start of a symbol, force a fresh packet by resetting any + // in-progress bundle state. This prevents packets from straddling label + // boundaries when data (e.g. jump tables) appears in between. + Size = 0; + resetBundle(); + return true; +} + static DecodeStatus DecodeRegisterClass(MCInst &Inst, unsigned RegNo, ArrayRef<MCPhysReg> Table) { if (RegNo < Table.size()) { diff --git a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp index a9201460d8e2..2399e2a28eb2 100644 --- a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp +++ b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp @@ -921,6 +921,10 @@ void HexagonGenInsert::collectInBlock(MachineBasicBlock *B, // successors have been processed. RegisterSet BlockDefs, InsDefs; for (MachineInstr &MI : *B) { + // Stop if the map size is too large. + if (IFMap.size() >= MaxIFMSize) + break; + InsDefs.clear(); getInstrDefs(&MI, InsDefs); // Leave those alone. They are more transparent than "insert". @@ -943,8 +947,8 @@ void HexagonGenInsert::collectInBlock(MachineBasicBlock *B, findRecordInsertForms(VR, AVs); // Stop if the map size is too large. - if (IFMap.size() > MaxIFMSize) - return; + if (IFMap.size() >= MaxIFMSize) + break; } } diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp index 53943de3bc59..e285e0454369 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp @@ -1640,6 +1640,15 @@ bool HexagonDAGToDAGISel::DetectUseSxtw(SDValue &N, SDValue &R) { R = N; break; } + case ISD::AssertSext: { + EVT T = cast<VTSDNode>(N.getOperand(1))->getVT(); + if (T.getSizeInBits() == 32) + R = N.getOperand(0); + else + return false; + break; + } + default: return false; } diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp index f1fa40c1b903..48918fa00ae0 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp @@ -355,6 +355,8 @@ HexagonTargetLowering::initializeHVXLowering() { setCondCodeAction(ISD::SETULE, MVT::v64f16, Expand); setCondCodeAction(ISD::SETUGE, MVT::v64f16, Expand); setCondCodeAction(ISD::SETULT, MVT::v64f16, Expand); + setCondCodeAction(ISD::SETUO, MVT::v64f16, Expand); + setCondCodeAction(ISD::SETO, MVT::v64f16, Expand); setCondCodeAction(ISD::SETNE, MVT::v32f32, Expand); setCondCodeAction(ISD::SETLE, MVT::v32f32, Expand); @@ -368,6 +370,8 @@ HexagonTargetLowering::initializeHVXLowering() { setCondCodeAction(ISD::SETULE, MVT::v32f32, Expand); setCondCodeAction(ISD::SETUGE, MVT::v32f32, Expand); setCondCodeAction(ISD::SETULT, MVT::v32f32, Expand); + setCondCodeAction(ISD::SETUO, MVT::v32f32, Expand); + setCondCodeAction(ISD::SETO, MVT::v32f32, Expand); // Boolean vectors. diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp index 64bc5ca134c8..35863f790eae 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp @@ -2803,6 +2803,7 @@ bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset, case Hexagon::V6_vL32b_nt_cur_npred_ai: case Hexagon::V6_vL32b_nt_tmp_pred_ai: case Hexagon::V6_vL32b_nt_tmp_npred_ai: + case Hexagon::V6_vS32Ub_npred_ai: case Hexagon::V6_vgathermh_pseudo: case Hexagon::V6_vgathermw_pseudo: case Hexagon::V6_vgathermhw_pseudo: diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp index 9030e43b7149..f83e06cd3d93 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp @@ -33,30 +33,18 @@ void HexagonInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) { void HexagonInstPrinter::printInst(const MCInst *MI, uint64_t Address, StringRef Annot, const MCSubtargetInfo &STI, raw_ostream &OS) { - assert(HexagonMCInstrInfo::isBundle(*MI)); - assert(HexagonMCInstrInfo::bundleSize(*MI) <= HEXAGON_PACKET_SIZE); - assert(HexagonMCInstrInfo::bundleSize(*MI) > 0); - HasExtender = false; - for (auto const &I : HexagonMCInstrInfo::bundleInstructions(*MI)) { - MCInst const &MCI = *I.getInst(); - if (HexagonMCInstrInfo::isDuplex(MII, MCI)) { - printInstruction(MCI.getOperand(1).getInst(), Address, OS); - OS << '\v'; - HasExtender = false; - printInstruction(MCI.getOperand(0).getInst(), Address, OS); - } else - printInstruction(&MCI, Address, OS); - HasExtender = HexagonMCInstrInfo::isImmext(MCI); - OS << "\n"; - } - - bool IsLoop0 = HexagonMCInstrInfo::isInnerLoop(*MI); - bool IsLoop1 = HexagonMCInstrInfo::isOuterLoop(*MI); - if (IsLoop0) { - OS << (IsLoop1 ? " :endloop01" : " :endloop0"); - } else if (IsLoop1) { - OS << " :endloop1"; + if (HexagonMCInstrInfo::isDuplex(MII, *MI)) { + printInstruction(MI->getOperand(1).getInst(), Address, OS); + OS << '\v'; + HasExtender = false; + printInstruction(MI->getOperand(0).getInst(), Address, OS); + } else { + printInstruction(MI, Address, OS); } + HasExtender = HexagonMCInstrInfo::isImmext(*MI); + if ((MI->getOpcode() & HexagonII::INST_PARSE_MASK) == + HexagonII::INST_PARSE_PACKET_END) + HasExtender = false; } void HexagonInstPrinter::printOperand(MCInst const *MI, unsigned OpNo, diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp index 980df819b2c2..bfea50e2d6dc 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp @@ -252,8 +252,21 @@ public: std::string Buffer; { raw_string_ostream TempStream(Buffer); - InstPrinter.printInst(&Inst, Address, "", STI, TempStream); + for (auto &I : HexagonMCInstrInfo::bundleInstructions(Inst)) { + InstPrinter.printInst(I.getInst(), Address, "", STI, TempStream); + TempStream << "\n"; + } + } + + std::string LoopString = ""; + bool IsLoop0 = HexagonMCInstrInfo::isInnerLoop(Inst); + bool IsLoop1 = HexagonMCInstrInfo::isOuterLoop(Inst); + if (IsLoop0) { + LoopString += (IsLoop1 ? " :endloop01" : " :endloop0"); + } else if (IsLoop1) { + LoopString += " :endloop1"; } + StringRef Contents(Buffer); auto PacketBundle = Contents.rsplit('\n'); auto HeadTail = PacketBundle.first.split('\n'); @@ -275,9 +288,9 @@ public: } if (HexagonMCInstrInfo::isMemReorderDisabled(Inst)) - OS << "\n\t} :mem_noshuf" << PacketBundle.second; + OS << "\n\t} :mem_noshuf" << LoopString; else - OS << "\t}" << PacketBundle.second; + OS << "\t}" << LoopString; } void finish() override { finishAttributeSection(); } diff --git a/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp b/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp index b757d123fa0f..2ab2a98c9434 100644 --- a/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp @@ -162,7 +162,7 @@ bool LoongArchAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, else if (OffsetMO.isImm()) OS << ", " << OffsetMO.getImm(); else if (OffsetMO.isGlobal() || OffsetMO.isBlockAddress() || - OffsetMO.isMCSymbol()) { + OffsetMO.isMCSymbol() || OffsetMO.isCPI()) { OS << ", "; MAI->printExpr(OS, *MCO.getExpr()); } else diff --git a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp index ac5e7f3891c7..1493bf4cba69 100644 --- a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp @@ -158,7 +158,12 @@ void LoongArchFrameLowering::processFunctionBeforeFrameFinalized( // estimateStackSize has been observed to under-estimate the final stack // size, so give ourselves wiggle-room by checking for stack size // representable an 11-bit signed field rather than 12-bits. - if (!isInt<11>(MFI.estimateStackSize(MF))) + // For [x]vstelm.{b/h/w/d} memory instructions with 8 imm offset, 7-bit + // signed field is fine. + unsigned EstimateStackSize = MFI.estimateStackSize(MF); + if (!isInt<11>(EstimateStackSize) || + (MF.getSubtarget<LoongArchSubtarget>().hasExtLSX() && + !isInt<7>(EstimateStackSize))) ScavSlotsNum = std::max(ScavSlotsNum, 1u); // For CFR spill. diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index c47987fbf683..d6adcf37f06f 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -2430,11 +2430,14 @@ static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp, } // make sure that this load is valid and only has one user. - if (!IdentitySrc || !BVOp->isOnlyUserOf(IdentitySrc.getNode())) + if (!IsIdeneity || !IdentitySrc || !BVOp->isOnlyUserOf(IdentitySrc.getNode())) return SDValue(); - if (IsIdeneity) { - auto *LN = cast<LoadSDNode>(IdentitySrc); + auto *LN = cast<LoadSDNode>(IdentitySrc); + auto ExtType = LN->getExtensionType(); + + if ((ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) && + VT.getScalarSizeInBits() == LN->getMemoryVT().getScalarSizeInBits()) { SDVTList Tys = LN->isIndexed() ? DAG.getVTList(VT, LN->getBasePtr().getValueType(), MVT::Other) @@ -4563,6 +4566,80 @@ static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT, llvm_unreachable("Unexpected node type for vXi1 sign extension"); } +static SDValue +performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG, + TargetLowering::DAGCombinerInfo &DCI, + const LoongArchSubtarget &Subtarget) { + SDLoc DL(N); + EVT VT = N->getValueType(0); + SDValue Src = N->getOperand(0); + EVT SrcVT = Src.getValueType(); + + if (Src.getOpcode() != ISD::SETCC || !Src.hasOneUse()) + return SDValue(); + + bool UseLASX; + unsigned Opc = ISD::DELETED_NODE; + EVT CmpVT = Src.getOperand(0).getValueType(); + EVT EltVT = CmpVT.getVectorElementType(); + + if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() == 128) + UseLASX = false; + else if (Subtarget.has32S() && Subtarget.hasExtLASX() && + CmpVT.getSizeInBits() == 256) + UseLASX = true; + else + return SDValue(); + + SDValue SrcN1 = Src.getOperand(1); + switch (cast<CondCodeSDNode>(Src.getOperand(2))->get()) { + default: + break; + case ISD::SETEQ: + // x == 0 => not (vmsknez.b x) + if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8) + Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ; + break; + case ISD::SETGT: + // x > -1 => vmskgez.b x + if (ISD::isBuildVectorAllOnes(SrcN1.getNode()) && EltVT == MVT::i8) + Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ; + break; + case ISD::SETGE: + // x >= 0 => vmskgez.b x + if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8) + Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ; + break; + case ISD::SETLT: + // x < 0 => vmskltz.{b,h,w,d} x + if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && + (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 || + EltVT == MVT::i64)) + Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ; + break; + case ISD::SETLE: + // x <= -1 => vmskltz.{b,h,w,d} x + if (ISD::isBuildVectorAllOnes(SrcN1.getNode()) && + (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 || + EltVT == MVT::i64)) + Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ; + break; + case ISD::SETNE: + // x != 0 => vmsknez.b x + if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8) + Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ; + break; + } + + if (Opc == ISD::DELETED_NODE) + return SDValue(); + + SDValue V = DAG.getNode(Opc, DL, MVT::i64, Src.getOperand(0)); + EVT T = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements()); + V = DAG.getZExtOrTrunc(V, DL, T); + return DAG.getBitcast(VT, V); +} + static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget) { @@ -4577,110 +4654,63 @@ static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG, if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1) return SDValue(); - unsigned Opc = ISD::DELETED_NODE; // Combine SETCC and BITCAST into [X]VMSK{LT,GE,NE} when possible + SDValue Res = performSETCC_BITCASTCombine(N, DAG, DCI, Subtarget); + if (Res) + return Res; + + // Generate vXi1 using [X]VMSKLTZ + MVT SExtVT; + unsigned Opc; + bool UseLASX = false; + bool PropagateSExt = false; + if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) { - bool UseLASX; EVT CmpVT = Src.getOperand(0).getValueType(); - EVT EltVT = CmpVT.getVectorElementType(); - - if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() <= 128) - UseLASX = false; - else if (Subtarget.has32S() && Subtarget.hasExtLASX() && - CmpVT.getSizeInBits() <= 256) - UseLASX = true; - else + if (CmpVT.getSizeInBits() > 256) return SDValue(); - - SDValue SrcN1 = Src.getOperand(1); - switch (cast<CondCodeSDNode>(Src.getOperand(2))->get()) { - default: - break; - case ISD::SETEQ: - // x == 0 => not (vmsknez.b x) - if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8) - Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ; - break; - case ISD::SETGT: - // x > -1 => vmskgez.b x - if (ISD::isBuildVectorAllOnes(SrcN1.getNode()) && EltVT == MVT::i8) - Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ; - break; - case ISD::SETGE: - // x >= 0 => vmskgez.b x - if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8) - Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ; - break; - case ISD::SETLT: - // x < 0 => vmskltz.{b,h,w,d} x - if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && - (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 || - EltVT == MVT::i64)) - Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ; - break; - case ISD::SETLE: - // x <= -1 => vmskltz.{b,h,w,d} x - if (ISD::isBuildVectorAllOnes(SrcN1.getNode()) && - (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 || - EltVT == MVT::i64)) - Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ; - break; - case ISD::SETNE: - // x != 0 => vmsknez.b x - if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8) - Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ; - break; - } } - // Generate vXi1 using [X]VMSKLTZ - if (Opc == ISD::DELETED_NODE) { - MVT SExtVT; - bool UseLASX = false; - bool PropagateSExt = false; - switch (SrcVT.getSimpleVT().SimpleTy) { - default: - return SDValue(); - case MVT::v2i1: - SExtVT = MVT::v2i64; - break; - case MVT::v4i1: - SExtVT = MVT::v4i32; - if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) { - SExtVT = MVT::v4i64; - UseLASX = true; - PropagateSExt = true; - } - break; - case MVT::v8i1: - SExtVT = MVT::v8i16; - if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) { - SExtVT = MVT::v8i32; - UseLASX = true; - PropagateSExt = true; - } - break; - case MVT::v16i1: - SExtVT = MVT::v16i8; - if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) { - SExtVT = MVT::v16i16; - UseLASX = true; - PropagateSExt = true; - } - break; - case MVT::v32i1: - SExtVT = MVT::v32i8; + switch (SrcVT.getSimpleVT().SimpleTy) { + default: + return SDValue(); + case MVT::v2i1: + SExtVT = MVT::v2i64; + break; + case MVT::v4i1: + SExtVT = MVT::v4i32; + if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) { + SExtVT = MVT::v4i64; UseLASX = true; - break; - }; - if (UseLASX && !Subtarget.has32S() && !Subtarget.hasExtLASX()) - return SDValue(); - Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL) - : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src); - Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ; - } else { - Src = Src.getOperand(0); - } + PropagateSExt = true; + } + break; + case MVT::v8i1: + SExtVT = MVT::v8i16; + if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) { + SExtVT = MVT::v8i32; + UseLASX = true; + PropagateSExt = true; + } + break; + case MVT::v16i1: + SExtVT = MVT::v16i8; + if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) { + SExtVT = MVT::v16i16; + UseLASX = true; + PropagateSExt = true; + } + break; + case MVT::v32i1: + SExtVT = MVT::v32i8; + UseLASX = true; + break; + }; + if (UseLASX && !(Subtarget.has32S() && Subtarget.hasExtLASX())) + return SDValue(); + Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL) + : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src); + Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ; SDValue V = DAG.getNode(Opc, DL, MVT::i64, Src); EVT T = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements()); diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp index 26d36f1c5058..75a230268bf5 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp @@ -378,12 +378,9 @@ bool LoongArchInstrInfo::isBranchOffsetInRange(unsigned BranchOp, } } -bool LoongArchInstrInfo::isSchedulingBoundary(const MachineInstr &MI, - const MachineBasicBlock *MBB, - const MachineFunction &MF) const { - if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF)) - return true; - +bool LoongArchInstrInfo::isSafeToMove(const MachineInstr &MI, + const MachineBasicBlock *MBB, + const MachineFunction &MF) const { auto MII = MI.getIterator(); auto MIE = MBB->end(); @@ -429,25 +426,25 @@ bool LoongArchInstrInfo::isSchedulingBoundary(const MachineInstr &MI, auto MO2 = Lu32I->getOperand(2).getTargetFlags(); if (MO0 == LoongArchII::MO_PCREL_HI && MO1 == LoongArchII::MO_PCREL_LO && MO2 == LoongArchII::MO_PCREL64_LO) - return true; + return false; if ((MO0 == LoongArchII::MO_GOT_PC_HI || MO0 == LoongArchII::MO_LD_PC_HI || MO0 == LoongArchII::MO_GD_PC_HI) && MO1 == LoongArchII::MO_GOT_PC_LO && MO2 == LoongArchII::MO_GOT_PC64_LO) - return true; + return false; if (MO0 == LoongArchII::MO_IE_PC_HI && MO1 == LoongArchII::MO_IE_PC_LO && MO2 == LoongArchII::MO_IE_PC64_LO) - return true; + return false; if (MO0 == LoongArchII::MO_DESC_PC_HI && MO1 == LoongArchII::MO_DESC_PC_LO && MO2 == LoongArchII::MO_DESC64_PC_LO) - return true; + return false; break; } case LoongArch::LU52I_D: { auto MO = MI.getOperand(2).getTargetFlags(); if (MO == LoongArchII::MO_PCREL64_HI || MO == LoongArchII::MO_GOT_PC64_HI || MO == LoongArchII::MO_IE_PC64_HI || MO == LoongArchII::MO_DESC64_PC_HI) - return true; + return false; break; } default: @@ -487,7 +484,7 @@ bool LoongArchInstrInfo::isSchedulingBoundary(const MachineInstr &MI, auto MO1 = LoongArchII::getDirectFlags(SecondOp->getOperand(2)); auto MO2 = LoongArchII::getDirectFlags(Ld->getOperand(2)); if (MO1 == LoongArchII::MO_DESC_PC_LO && MO2 == LoongArchII::MO_DESC_LD) - return true; + return false; break; } if (SecondOp == MIE || @@ -496,34 +493,34 @@ bool LoongArchInstrInfo::isSchedulingBoundary(const MachineInstr &MI, auto MO1 = LoongArchII::getDirectFlags(SecondOp->getOperand(2)); if (MO0 == LoongArchII::MO_PCREL_HI && SecondOp->getOpcode() == AddiOp && MO1 == LoongArchII::MO_PCREL_LO) - return true; + return false; if (MO0 == LoongArchII::MO_GOT_PC_HI && SecondOp->getOpcode() == LdOp && MO1 == LoongArchII::MO_GOT_PC_LO) - return true; + return false; if ((MO0 == LoongArchII::MO_LD_PC_HI || MO0 == LoongArchII::MO_GD_PC_HI) && SecondOp->getOpcode() == AddiOp && MO1 == LoongArchII::MO_GOT_PC_LO) - return true; + return false; break; } case LoongArch::ADDI_W: case LoongArch::ADDI_D: { auto MO = LoongArchII::getDirectFlags(MI.getOperand(2)); if (MO == LoongArchII::MO_PCREL_LO || MO == LoongArchII::MO_GOT_PC_LO) - return true; + return false; break; } case LoongArch::LD_W: case LoongArch::LD_D: { auto MO = LoongArchII::getDirectFlags(MI.getOperand(2)); if (MO == LoongArchII::MO_GOT_PC_LO) - return true; + return false; break; } case LoongArch::PseudoDESC_CALL: { auto MO = LoongArchII::getDirectFlags(MI.getOperand(2)); if (MO == LoongArchII::MO_DESC_CALL) - return true; + return false; break; } default: @@ -531,6 +528,18 @@ bool LoongArchInstrInfo::isSchedulingBoundary(const MachineInstr &MI, } } + return true; +} + +bool LoongArchInstrInfo::isSchedulingBoundary(const MachineInstr &MI, + const MachineBasicBlock *MBB, + const MachineFunction &MF) const { + if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF)) + return true; + + if (!isSafeToMove(MI, MBB, MF)) + return true; + return false; } diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h index 63b7112b8b40..282ea30b0329 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.h @@ -64,6 +64,9 @@ public: bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override; + bool isSafeToMove(const MachineInstr &MI, const MachineBasicBlock *MBB, + const MachineFunction &MF) const override; + bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override; diff --git a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp index f62753356a4d..a2292d016efa 100644 --- a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp @@ -759,6 +759,10 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20, MO.ChangeToBA(ImmOp.getBlockAddress(), ImmOp.getOffset(), LoongArchII::getDirectFlags(ImmOp)); break; + case MachineOperand::MO_ConstantPoolIndex: + MO.ChangeToCPI(ImmOp.getIndex(), ImmOp.getOffset(), + LoongArchII::getDirectFlags(ImmOp)); + break; default: report_fatal_error("unsupported machine operand type"); break; diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp index 03ce004ed33a..7cefb3f8119b 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp @@ -52,6 +52,9 @@ static ABI getTripleABI(const Triple &TT) { bool Is64Bit = TT.isArch64Bit(); ABI TripleABI; switch (TT.getEnvironment()) { + case llvm::Triple::EnvironmentType::UnknownEnvironment: + TripleABI = ABI_Unknown; + break; case llvm::Triple::EnvironmentType::GNUSF: case llvm::Triple::EnvironmentType::MuslSF: TripleABI = Is64Bit ? ABI_LP64S : ABI_ILP32S; @@ -96,7 +99,7 @@ ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits, // 1. If the '-target-abi' is valid, use it. if (IsABIValidForFeature(ArgProvidedABI)) { - if (TT.hasEnvironment() && ArgProvidedABI != TripleABI) + if (IsABIValidForFeature(TripleABI) && ArgProvidedABI != TripleABI) errs() << "warning: triple-implied ABI conflicts with provided target-abi '" << ABIName << "', using target-abi\n"; @@ -164,10 +167,7 @@ ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits, return Is64Bit ? ABI_LP64F : ABI_ILP32F; return Is64Bit ? ABI_LP64S : ABI_ILP32S; }; - if (ABIName.empty()) - errs() << "warning: the triple-implied ABI is invalid, ignoring and using " - "feature-implied ABI\n"; - else + if (!ABIName.empty()) errs() << "warning: both target-abi and the triple-implied ABI are " "invalid, ignoring and using feature-implied ABI\n"; return checkABIStandardized(GetFeatureABI()); diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index 01e4d17f6236..602b89a11759 100644 --- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -3676,7 +3676,7 @@ bool MipsAsmParser::expandBranchImm(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out, Out, STI)) return true; - if (IsLikely) { + if (IsLikely && MemOffsetOp.isExpr()) { TOut.emitRRX(OpCode, DstRegOp.getReg(), ATReg, MCOperand::createExpr(MemOffsetOp.getExpr()), IDLoc, STI); TOut.emitRRI(Mips::SLL, Mips::ZERO, Mips::ZERO, 0, IDLoc, STI); diff --git a/llvm/lib/Target/Mips/MipsAsmPrinter.cpp b/llvm/lib/Target/Mips/MipsAsmPrinter.cpp index 87e06a6d3c08..2903ff75475c 100644 --- a/llvm/lib/Target/Mips/MipsAsmPrinter.cpp +++ b/llvm/lib/Target/Mips/MipsAsmPrinter.cpp @@ -747,14 +747,18 @@ void MipsAsmPrinter::emitStartOfAsmFile(Module &M) { if (FS.empty() && M.size() && F->hasFnAttribute("target-features")) FS = F->getFnAttribute("target-features").getValueAsString(); + std::string strFS = FS.str(); + if (M.size() && F->getFnAttribute("use-soft-float").getValueAsBool()) + strFS += strFS.empty() ? "+soft-float" : ",+soft-float"; + // Compute MIPS architecture attributes based on the default subtarget // that we'd have constructed. // FIXME: For ifunc related functions we could iterate over and look // for a feature string that doesn't match the default one. StringRef CPU = MIPS_MC::selectMipsCPU(TT, TM.getTargetCPU()); const MipsTargetMachine &MTM = static_cast<const MipsTargetMachine &>(TM); - const MipsSubtarget STI(TT, CPU, FS, MTM.isLittleEndian(), MTM, - std::nullopt); + const MipsSubtarget STI(TT, CPU, StringRef(strFS), MTM.isLittleEndian(), + MTM, std::nullopt); bool IsABICalls = STI.isABICalls(); const MipsABIInfo &ABI = MTM.getABI(); diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp index 34ff41f6e02d..78f2e5db40f9 100644 --- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp +++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp @@ -432,13 +432,24 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( Register OldVal = I->getOperand(6).getReg(); Register BinOpRes = I->getOperand(7).getReg(); Register StoreVal = I->getOperand(8).getReg(); + bool NoMovnInstr = (IsMin || IsMax) && !STI->hasMips4() && !STI->hasMips32(); const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *loop1MBB = nullptr; + MachineBasicBlock *loop2MBB = nullptr; + if (NoMovnInstr) { + loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); + loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); + } MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loopMBB); + if (NoMovnInstr) { + MF->insert(It, loop1MBB); + MF->insert(It, loop2MBB); + } MF->insert(It, sinkMBB); MF->insert(It, exitMBB); @@ -446,9 +457,19 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( exitMBB->transferSuccessorsAndUpdatePHIs(&BB); BB.addSuccessor(loopMBB, BranchProbability::getOne()); - loopMBB->addSuccessor(sinkMBB); - loopMBB->addSuccessor(loopMBB); - loopMBB->normalizeSuccProbs(); + if (NoMovnInstr) { + loopMBB->addSuccessor(loop1MBB); + loopMBB->addSuccessor(loop2MBB); + } else { + loopMBB->addSuccessor(sinkMBB); + loopMBB->addSuccessor(loopMBB); + loopMBB->normalizeSuccProbs(); + } + if (NoMovnInstr) { + loop1MBB->addSuccessor(loop2MBB); + loop2MBB->addSuccessor(loopMBB); + loop2MBB->addSuccessor(sinkMBB); + } BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); if (IsNand) { @@ -525,7 +546,7 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( BuildMI(loopMBB, DL, TII->get(OR), BinOpRes) .addReg(BinOpRes) .addReg(Scratch4); - } else { + } else if (STI->hasMips4() || STI->hasMips32()) { // max: move BinOpRes, StoreVal // movn BinOpRes, Incr, Scratch4, BinOpRes // min: move BinOpRes, StoreVal @@ -537,12 +558,59 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( .addReg(Incr) .addReg(Scratch4) .addReg(BinOpRes); + } else { + // if min: + // loopMBB: move BinOpRes, StoreVal + // beq Scratch4, 0, loop1MBB + // j loop2MBB + // loop1MBB: move BinOpRes, Incr + // loop2MBB: and BinOpRes, BinOpRes, Mask + // and StoreVal, OlddVal, Mask2 + // or StoreVal, StoreVal, BinOpRes + // StoreVal<tied1> = sc StoreVal, 0(Ptr) + // beq StoreVal, zero, loopMBB + // + // if max: + // loopMBB: move BinOpRes, Incr + // beq Scratch4, 0, loop1MBB + // j loop2MBB + // loop1MBB: move BinOpRes, StoreVal + // loop2MBB: and BinOpRes, BinOpRes, Mask + // and StoreVal, OlddVal, Mask2 + // or StoreVal, StoreVal, BinOpRes + // StoreVal<tied1> = sc StoreVal, 0(Ptr) + // beq StoreVal, zero, loopMBB + if (IsMin) { + BuildMI(loopMBB, DL, TII->get(OR), BinOpRes) + .addReg(StoreVal) + .addReg(Mips::ZERO); + BuildMI(loop1MBB, DL, TII->get(OR), BinOpRes) + .addReg(Incr) + .addReg(Mips::ZERO); + } else { + BuildMI(loopMBB, DL, TII->get(OR), BinOpRes) + .addReg(Incr) + .addReg(Mips::ZERO); + BuildMI(loop1MBB, DL, TII->get(OR), BinOpRes) + .addReg(StoreVal) + .addReg(Mips::ZERO); + } + BuildMI(loopMBB, DL, TII->get(BEQ)) + .addReg(Scratch4) + .addReg(Mips::ZERO) + .addMBB(loop1MBB); + BuildMI(loopMBB, DL, TII->get(Mips::J)).addMBB(loop2MBB); } // and BinOpRes, BinOpRes, Mask - BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) - .addReg(BinOpRes) - .addReg(Mask); + if (NoMovnInstr) + BuildMI(loop2MBB, DL, TII->get(Mips::AND), BinOpRes) + .addReg(BinOpRes) + .addReg(Mask); + else + BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) + .addReg(BinOpRes) + .addReg(Mask); } else if (!IsSwap) { // <binop> binopres, oldval, incr2 @@ -564,14 +632,37 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( // or StoreVal, StoreVal, BinOpRes // StoreVal<tied1> = sc StoreVal, 0(Ptr) // beq StoreVal, zero, loopMBB - BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal) - .addReg(OldVal).addReg(Mask2); - BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal) - .addReg(StoreVal).addReg(BinOpRes); - BuildMI(loopMBB, DL, TII->get(SC), StoreVal) - .addReg(StoreVal).addReg(Ptr).addImm(0); - BuildMI(loopMBB, DL, TII->get(BEQ)) - .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB); + if (NoMovnInstr) { + BuildMI(loop2MBB, DL, TII->get(Mips::AND), StoreVal) + .addReg(OldVal) + .addReg(Mask2); + BuildMI(loop2MBB, DL, TII->get(Mips::OR), StoreVal) + .addReg(StoreVal) + .addReg(BinOpRes); + BuildMI(loop2MBB, DL, TII->get(SC), StoreVal) + .addReg(StoreVal) + .addReg(Ptr) + .addImm(0); + BuildMI(loop2MBB, DL, TII->get(BEQ)) + .addReg(StoreVal) + .addReg(Mips::ZERO) + .addMBB(loopMBB); + } else { + BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal) + .addReg(OldVal) + .addReg(Mask2); + BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal) + .addReg(StoreVal) + .addReg(BinOpRes); + BuildMI(loopMBB, DL, TII->get(SC), StoreVal) + .addReg(StoreVal) + .addReg(Ptr) + .addImm(0); + BuildMI(loopMBB, DL, TII->get(BEQ)) + .addReg(StoreVal) + .addReg(Mips::ZERO) + .addMBB(loopMBB); + } // sinkMBB: // and maskedoldval1,oldval,mask @@ -600,6 +691,11 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loopMBB); + if (loop1MBB) { + assert(loop2MBB && "should have 2 loop blocks"); + computeAndAddLiveIns(LiveRegs, *loop1MBB); + computeAndAddLiveIns(LiveRegs, *loop2MBB); + } computeAndAddLiveIns(LiveRegs, *sinkMBB); computeAndAddLiveIns(LiveRegs, *exitMBB); @@ -746,20 +842,41 @@ bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, llvm_unreachable("Unknown pseudo atomic!"); } + bool NoMovnInstr = (IsMin || IsMax) && !STI->hasMips4() && !STI->hasMips32(); const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); + MachineBasicBlock *loop1MBB = nullptr; + MachineBasicBlock *loop2MBB = nullptr; + if (NoMovnInstr) { + loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); + loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); + } MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loopMBB); + if (NoMovnInstr) { + MF->insert(It, loop1MBB); + MF->insert(It, loop2MBB); + } MF->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); BB.addSuccessor(loopMBB, BranchProbability::getOne()); - loopMBB->addSuccessor(exitMBB); - loopMBB->addSuccessor(loopMBB); + if (NoMovnInstr) { + loopMBB->addSuccessor(loop1MBB); + loopMBB->addSuccessor(loop2MBB); + } else { + loopMBB->addSuccessor(exitMBB); + loopMBB->addSuccessor(loopMBB); + } loopMBB->normalizeSuccProbs(); + if (NoMovnInstr) { + loop1MBB->addSuccessor(loop2MBB); + loop2MBB->addSuccessor(loopMBB); + loop2MBB->addSuccessor(exitMBB); + } BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!"); @@ -802,7 +919,7 @@ bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, BuildMI(loopMBB, DL, TII->get(OR), Scratch) .addReg(Scratch) .addReg(Scratch2); - } else { + } else if (STI->hasMips4() || STI->hasMips32()) { // max: move Scratch, OldVal // movn Scratch, Incr, Scratch2, Scratch // min: move Scratch, OldVal @@ -814,6 +931,38 @@ bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, .addReg(Incr) .addReg(Scratch2) .addReg(Scratch); + } else { + // if min: + // loopMBB: move Scratch, OldVal + // beq Scratch2_32, 0, loop1MBB + // j loop2MBB + // loop1MBB: move Scratch, Incr + // loop2MBB: sc $2, 0($4) + // beqz $2, $BB0_1 + // nop + // + // if max: + // loopMBB: move Scratch, Incr + // beq Scratch2_32, 0, loop1MBB + // j loop2MBB + // loop1MBB: move Scratch, OldVal + // loop2MBB: sc $2, 0($4) + // beqz $2, $BB0_1 + // nop + if (IsMin) { + BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(OldVal).addReg(ZERO); + BuildMI(loop1MBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); + } else { + BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); + BuildMI(loop1MBB, DL, TII->get(OR), Scratch) + .addReg(OldVal) + .addReg(ZERO); + } + BuildMI(loopMBB, DL, TII->get(BEQ)) + .addReg(Scratch2_32) + .addReg(ZERO) + .addMBB(loop1MBB); + BuildMI(loopMBB, DL, TII->get(Mips::J)).addMBB(loop2MBB); } } else if (Opcode) { @@ -829,20 +978,36 @@ bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); } - BuildMI(loopMBB, DL, TII->get(SC), Scratch) - .addReg(Scratch) - .addReg(Ptr) - .addImm(0); - BuildMI(loopMBB, DL, TII->get(BEQ)) - .addReg(Scratch) - .addReg(ZERO) - .addMBB(loopMBB); + if (NoMovnInstr) { + BuildMI(loop2MBB, DL, TII->get(SC), Scratch) + .addReg(Scratch) + .addReg(Ptr) + .addImm(0); + BuildMI(loop2MBB, DL, TII->get(BEQ)) + .addReg(Scratch) + .addReg(ZERO) + .addMBB(loopMBB); + } else { + BuildMI(loopMBB, DL, TII->get(SC), Scratch) + .addReg(Scratch) + .addReg(Ptr) + .addImm(0); + BuildMI(loopMBB, DL, TII->get(BEQ)) + .addReg(Scratch) + .addReg(ZERO) + .addMBB(loopMBB); + } NMBBI = BB.end(); I->eraseFromParent(); LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loopMBB); + if (loop1MBB) { + assert(loop2MBB && "should have 2 loop blocks"); + computeAndAddLiveIns(LiveRegs, *loop1MBB); + computeAndAddLiveIns(LiveRegs, *loop2MBB); + } computeAndAddLiveIns(LiveRegs, *exitMBB); return true; diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp index 0e581a7a1650..ec6b38215166 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -522,9 +522,6 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM, setOperationAction(ISD::TRAP, MVT::Other, Legal); - setOperationAction(ISD::ConstantFP, MVT::f32, Custom); - setOperationAction(ISD::ConstantFP, MVT::f64, Custom); - setTargetDAGCombine({ISD::SDIVREM, ISD::UDIVREM, ISD::SELECT, ISD::AND, ISD::OR, ISD::ADD, ISD::SUB, ISD::AssertZext, ISD::SHL, ISD::SIGN_EXTEND}); @@ -1360,8 +1357,6 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) const case ISD::FP_TO_SINT: return lowerFP_TO_SINT(Op, DAG); case ISD::READCYCLECOUNTER: return lowerREADCYCLECOUNTER(Op, DAG); - case ISD::ConstantFP: - return lowerConstantFP(Op, DAG); } return SDValue(); } @@ -3019,30 +3014,6 @@ SDValue MipsTargetLowering::lowerFP_TO_SINT(SDValue Op, return DAG.getNode(ISD::BITCAST, SDLoc(Op), Op.getValueType(), Trunc); } -SDValue MipsTargetLowering::lowerConstantFP(SDValue Op, - SelectionDAG &DAG) const { - SDLoc DL(Op); - EVT VT = Op.getSimpleValueType(); - SDNode *N = Op.getNode(); - ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(N); - - if (!CFP->isNaN() || Subtarget.isNaN2008()) { - return SDValue(); - } - - APFloat NaNValue = CFP->getValueAPF(); - auto &Sem = NaNValue.getSemantics(); - - // The MSB of the mantissa should be zero for QNaNs in the MIPS legacy NaN - // encodings, and one for sNaNs. Check every NaN constants and make sure - // they are correctly encoded for legacy encodings. - if (!NaNValue.isSignaling()) { - APFloat RealQNaN = NaNValue.getSNaN(Sem); - return DAG.getConstantFP(RealQNaN, DL, VT); - } - return SDValue(); -} - //===----------------------------------------------------------------------===// // Calling Convention Implementation //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h index 31ac5d4c185b..c65c76ccffc7 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.h +++ b/llvm/lib/Target/Mips/MipsISelLowering.h @@ -592,7 +592,6 @@ class TargetRegisterClass; SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const; SDValue lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const; SDValue lowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; - SDValue lowerConstantFP(SDValue Op, SelectionDAG &DAG) const; /// isEligibleForTailCallOptimization - Check whether the call is eligible /// for tail call optimization. diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.td b/llvm/lib/Target/Mips/MipsInstrInfo.td index b6125b972717..255fd838a72a 100644 --- a/llvm/lib/Target/Mips/MipsInstrInfo.td +++ b/llvm/lib/Target/Mips/MipsInstrInfo.td @@ -858,6 +858,16 @@ def calltarget : Operand<iPTR> { def imm64: Operand<i64>; +def ConstantImmAsmOperandClass : AsmOperandClass { + let Name = "ConstantImm"; + let PredicateMethod = "isConstantImm"; + let RenderMethod = "addImmOperands"; +} + +def ConstantImm64: Operand<i64> { + let ParserMatchClass = ConstantImmAsmOperandClass; +} + def simm19_lsl2 : Operand<i32> { let EncoderMethod = "getSimm19Lsl2Encoding"; let DecoderMethod = "DecodeSimm19Lsl2"; @@ -2950,10 +2960,10 @@ def : MipsInstAlias<"nor\t$rs, $imm", (NORImm GPR32Opnd:$rs, GPR32Opnd:$rs, let hasDelaySlot = 1, isCTI = 1 in { def BneImm : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), - (ins imm64:$imm64, brtarget:$offset), + (ins ConstantImm64:$imm64, brtarget:$offset), "bne\t$rt, $imm64, $offset">; def BeqImm : MipsAsmPseudoInst<(outs GPR32Opnd:$rt), - (ins imm64:$imm64, brtarget:$offset), + (ins ConstantImm64:$imm64, brtarget:$offset), "beq\t$rt, $imm64, $offset">; class CondBranchPseudo<string instr_asm> : @@ -2981,7 +2991,7 @@ def BGTUL: CondBranchPseudo<"bgtul">, ISA_MIPS2_NOT_32R6_64R6; let isCTI = 1 in class CondBranchImmPseudo<string instr_asm> : - MipsAsmPseudoInst<(outs), (ins GPR32Opnd:$rs, imm64:$imm, brtarget:$offset), + MipsAsmPseudoInst<(outs), (ins GPR32Opnd:$rs, ConstantImm64:$imm, brtarget:$offset), !strconcat(instr_asm, "\t$rs, $imm, $offset")>; def BEQLImmMacro : CondBranchImmPseudo<"beql">, ISA_MIPS2_NOT_32R6_64R6; diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 14f05250ad6b..5d496f69172c 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -5008,12 +5008,6 @@ combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { return !U.getUser()->use_empty(); } - // Handle CopyToReg nodes that will become dead after our replacement - if (U.getUser()->getOpcode() == ISD::CopyToReg) { - DeadCopyToRegs.push_back(U.getUser()); - return true; - } - // Otherwise, this use prevents us from splitting a value. return false; })) @@ -5080,10 +5074,6 @@ combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { for (unsigned I : seq(NewLoad->getNumValues() - NewNumOutputs)) Results.push_back(NewLoad.getValue(NewNumOutputs + I)); - // Remove dead CopyToReg nodes by folding them into the chain they reference - for (SDNode *CTR : DeadCopyToRegs) - DCI.CombineTo(CTR, CTR->getOperand(0)); - return DCI.DAG.getMergeValues(Results, DL); } @@ -6418,4 +6408,4 @@ void NVPTXTargetLowering::computeKnownBitsForTargetNode( default: break; } -}
\ No newline at end of file +} diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h index 9a6e261c811a..437edfb8ffc7 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h +++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h @@ -183,6 +183,11 @@ public: void collectKernelLaunchBounds( const Function &F, SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const override; + + bool shouldBuildRelLookupTables() const override { + // Self-referential globals are not supported. + return false; + } }; } // end namespace llvm diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 459525ed4ee9..b6db0abcc86d 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -7296,9 +7296,17 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( if (!ArgVT.isVector() && !ValVT.isVector() && ArgVT.isInteger() && ValVT.isInteger() && ArgVT.getScalarSizeInBits() < ValVT.getScalarSizeInBits()) { - SDValue ArgValueTrunc = DAG.getNode( - ISD::TRUNCATE, dl, ArgVT.getSimpleVT() == MVT::i1 ? MVT::i8 : ArgVT, - ArgValue); + // It is possible to have either real integer values + // or integers that were not originally integers. + // In the latter case, these could have came from structs, + // and these integers would not have an extend on the parameter. + // Since these types of integers do not have an extend specified + // in the first place, the type of extend that we do should not matter. + EVT TruncatedArgVT = ArgVT.isSimple() && ArgVT.getSimpleVT() == MVT::i1 + ? MVT::i8 + : ArgVT; + SDValue ArgValueTrunc = + DAG.getNode(ISD::TRUNCATE, dl, TruncatedArgVT, ArgValue); SDValue ArgValueExt = ArgSignExt ? DAG.getSExtOrTrunc(ArgValueTrunc, dl, ValVT) : DAG.getZExtOrTrunc(ArgValueTrunc, dl, ValVT); @@ -9586,12 +9594,14 @@ static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op, return false; } -bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN) { +bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN, + bool IsLittleEndian) { assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector"); BitMask.clearAllBits(); EVT VT = BVN.getValueType(0); - APInt ConstValue(VT.getSizeInBits(), 0); + unsigned VTSize = VT.getSizeInBits(); + APInt ConstValue(VTSize, 0); unsigned EltWidth = VT.getScalarSizeInBits(); @@ -9601,8 +9611,10 @@ bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN) { if (!CN) return false; - - ConstValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos); + // The elements in a vector register are ordered in reverse byte order + // between little-endian and big-endian modes. + ConstValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), + IsLittleEndian ? BitPos : VTSize - EltWidth - BitPos); BitPos += EltWidth; } @@ -9633,7 +9645,8 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, // we do not convert it to MTVSRBMI. // The xxleqv instruction sets a vector with all ones. // The xxlxor instruction sets a vector with all zeros. - if (isValidMtVsrBmi(BitMask, *BVN) && BitMask != 0 && BitMask != 0xffff) { + if (isValidMtVsrBmi(BitMask, *BVN, Subtarget.isLittleEndian()) && + BitMask != 0 && BitMask != 0xffff) { SDValue SDConstant = DAG.getTargetConstant(BitMask, dl, MVT::i32); MachineSDNode *MSDNode = DAG.getMachineNode(PPC::MTVSRBMI, dl, MVT::v16i8, SDConstant); @@ -15314,6 +15327,12 @@ SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, } } + // Convert PromOps to handles before doing any RAUW operations, as these + // may CSE with existing nodes, deleting the originals. + std::list<HandleSDNode> PromOpHandles; + for (auto &PromOp : PromOps) + PromOpHandles.emplace_back(PromOp); + // Replace all inputs, either with the truncation operand, or a // truncation or extension to the final output type. for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { @@ -15337,10 +15356,6 @@ SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); } - std::list<HandleSDNode> PromOpHandles; - for (auto &PromOp : PromOps) - PromOpHandles.emplace_back(PromOp); - // Replace all operations (these are all the same, but have a different // (promoted) return type). DAG.getNode will validate that the types of // a binary operator match, so go through the list in reverse so that diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 99ef89a7fdc0..dbbf06f47153 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -3252,7 +3252,8 @@ def PPC32GOT: PPCEmitTimePseudo<(outs gprc:$rD), (ins), "#PPC32GOT", // Get the _GLOBAL_OFFSET_TABLE_ in PIC mode. // This uses two output registers, the first as the real output, the second as a -// temporary register, used internally in code generation. +// temporary register, used internally in code generation. A "bl" also clobbers LR. +let Defs = [LR] in def PPC32PICGOT: PPCEmitTimePseudo<(outs gprc:$rD, gprc:$rT), (ins), "#PPC32PICGOT", []>, NoEncode<"$rT">; diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp index 76dca4794e05..f1230407b164 100644 --- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp @@ -1102,13 +1102,20 @@ void PPCRegisterInfo::lowerCRBitSpilling(MachineBasicBlock::iterator II, SpillsKnownBit = true; break; default: + // When spilling a CR bit, the super register may not be explicitly defined + // (i.e. it can be defined by a CR-logical that only defines the subreg) so + // we state that the CR field is undef. Also, in order to preserve the kill + // flag on the CR bit, we add it as an implicit use. + // On Power10, we can use SETNBC to spill all CR bits. SETNBC will set all // bits (specifically, it produces a -1 if the CR bit is set). Ultimately, // the bit that is of importance to us is bit 32 (bit 0 of a 32-bit // register), and SETNBC will set this. if (Subtarget.isISA3_1()) { BuildMI(MBB, II, dl, TII.get(LP64 ? PPC::SETNBC8 : PPC::SETNBC), Reg) - .addReg(SrcReg, RegState::Undef); + .addReg(SrcReg, RegState::Undef) + .addReg(SrcReg, RegState::Implicit | + getKillRegState(MI.getOperand(0).isKill())); break; } @@ -1122,16 +1129,14 @@ void PPCRegisterInfo::lowerCRBitSpilling(MachineBasicBlock::iterator II, SrcReg == PPC::CR4LT || SrcReg == PPC::CR5LT || SrcReg == PPC::CR6LT || SrcReg == PPC::CR7LT) { BuildMI(MBB, II, dl, TII.get(LP64 ? PPC::SETB8 : PPC::SETB), Reg) - .addReg(getCRFromCRBit(SrcReg), RegState::Undef); + .addReg(getCRFromCRBit(SrcReg), RegState::Undef) + .addReg(SrcReg, RegState::Implicit | + getKillRegState(MI.getOperand(0).isKill())); break; } } // We need to move the CR field that contains the CR bit we are spilling. - // The super register may not be explicitly defined (i.e. it can be defined - // by a CR-logical that only defines the subreg) so we state that the CR - // field is undef. Also, in order to preserve the kill flag on the CR bit, - // we add it as an implicit use. BuildMI(MBB, II, dl, TII.get(LP64 ? PPC::MFOCRF8 : PPC::MFOCRF), Reg) .addReg(getCRFromCRBit(SrcReg), RegState::Undef) .addReg(SrcReg, diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp index e42d6c539a34..df3028e05139 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp @@ -792,6 +792,23 @@ void RISCVAsmBackend::maybeAddVendorReloc(const MCFragment &F, Asm->getWriter().recordRelocation(F, VendorFixup, VendorTarget, VendorValue); } +static bool relaxableFixupNeedsRelocation(const MCFixupKind Kind) { + // Some Fixups are marked as LinkerRelaxable by + // `RISCVMCCodeEmitter::getImmOpValue` only because they may be + // (assembly-)relaxed into a linker-relaxable instruction. This function + // should return `false` for those fixups so they do not get a `R_RISCV_RELAX` + // relocation emitted in addition to the relocation. + switch (Kind) { + default: + break; + case RISCV::fixup_riscv_rvc_jump: + case RISCV::fixup_riscv_rvc_branch: + case RISCV::fixup_riscv_jal: + return false; + } + return true; +} + bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup, const MCValue &Target, uint64_t &FixedValue, bool IsResolved) { @@ -834,25 +851,32 @@ bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup, return false; } - // If linker relaxation is enabled and supported by the current relocation, - // generate a relocation and then append a RELAX. - if (Fixup.isLinkerRelaxable()) + // If linker relaxation is enabled and supported by the current fixup, then we + // always want to generate a relocation. + bool NeedsRelax = Fixup.isLinkerRelaxable() && + relaxableFixupNeedsRelocation(Fixup.getKind()); + if (NeedsRelax) IsResolved = false; + if (IsResolved && Fixup.isPCRel()) IsResolved = isPCRelFixupResolved(Target.getAddSym(), F); if (!IsResolved) { - // Some Fixups require a vendor relocation, record it (directly) before we + // Some Fixups require a VENDOR relocation, record it (directly) before we // add the relocation. maybeAddVendorReloc(F, Fixup); Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue); - } - if (Fixup.isLinkerRelaxable()) { - auto FA = MCFixup::create(Fixup.getOffset(), nullptr, ELF::R_RISCV_RELAX); - Asm->getWriter().recordRelocation(F, FA, MCValue::get(nullptr), - FixedValueA); + if (NeedsRelax) { + // Some Fixups get a RELAX relocation, record it (directly) after we add + // the relocation. + MCFixup RelaxFixup = + MCFixup::create(Fixup.getOffset(), nullptr, ELF::R_RISCV_RELAX); + MCValue RelaxTarget = MCValue::get(nullptr); + uint64_t RelaxValue; + Asm->getWriter().recordRelocation(F, RelaxFixup, RelaxTarget, RelaxValue); + } } return false; diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp index cbeabdddb937..717fba68b48e 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp @@ -576,8 +576,21 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo, "getImmOpValue expects only expressions or immediates"); const MCExpr *Expr = MO.getExpr(); MCExpr::ExprKind Kind = Expr->getKind(); - unsigned FixupKind = RISCV::fixup_riscv_invalid; + + // `RelaxCandidate` must be set to `true` in two cases: + // - The fixup's relocation gets a R_RISCV_RELAX relocation + // - The underlying instruction may be relaxed to an instruction that gets a + // `R_RISCV_RELAX` relocation. + // + // The actual emission of `R_RISCV_RELAX` will be handled in + // `RISCVAsmBackend::applyFixup`. bool RelaxCandidate = false; + auto AsmRelaxToLinkerRelaxableWithFeature = [&](unsigned Feature) -> void { + if (!STI.hasFeature(RISCV::FeatureExactAssembly) && STI.hasFeature(Feature)) + RelaxCandidate = true; + }; + + unsigned FixupKind = RISCV::fixup_riscv_invalid; if (Kind == MCExpr::Specifier) { const auto *RVExpr = cast<MCSpecifierExpr>(Expr); FixupKind = RVExpr->getSpecifier(); @@ -644,18 +657,26 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo, // FIXME: Sub kind binary exprs have chance of underflow. if (MIFrm == RISCVII::InstFormatJ) { FixupKind = RISCV::fixup_riscv_jal; + AsmRelaxToLinkerRelaxableWithFeature(RISCV::FeatureVendorXqcilb); } else if (MIFrm == RISCVII::InstFormatB) { FixupKind = RISCV::fixup_riscv_branch; + // This might be assembler relaxed to `b<cc>; jal` but we cannot relax + // the `jal` again in the assembler. } else if (MIFrm == RISCVII::InstFormatCJ) { FixupKind = RISCV::fixup_riscv_rvc_jump; + AsmRelaxToLinkerRelaxableWithFeature(RISCV::FeatureVendorXqcilb); } else if (MIFrm == RISCVII::InstFormatCB) { FixupKind = RISCV::fixup_riscv_rvc_branch; + // This might be assembler relaxed to `b<cc>; jal` but we cannot relax + // the `jal` again in the assembler. } else if (MIFrm == RISCVII::InstFormatCI) { FixupKind = RISCV::fixup_riscv_rvc_imm; } else if (MIFrm == RISCVII::InstFormatI) { FixupKind = RISCV::fixup_riscv_12_i; } else if (MIFrm == RISCVII::InstFormatQC_EB) { FixupKind = RISCV::fixup_riscv_qc_e_branch; + // This might be assembler relaxed to `qc.e.b<cc>; jal` but we cannot + // relax the `jal` again in the assembler. } else if (MIFrm == RISCVII::InstFormatQC_EAI) { FixupKind = RISCV::fixup_riscv_qc_e_32; RelaxCandidate = true; @@ -670,9 +691,9 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo, assert(FixupKind != RISCV::fixup_riscv_invalid && "Unhandled expression!"); addFixup(Fixups, 0, Expr, FixupKind); - // If linker relaxation is enabled and supported by this relocation, set - // a bit so that if fixup is unresolved, a R_RISCV_RELAX relocation will be - // appended. + // If linker relaxation is enabled and supported by this relocation, set a bit + // so that the assembler knows the size of the instruction is not fixed/known, + // and the relocation will need a R_RISCV_RELAX relocation. if (EnableRelax && RelaxCandidate) Fixups.back().setLinkerRelaxable(); ++MCNumFixups; diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.td b/llvm/lib/Target/RISCV/RISCVCallingConv.td index cbf039edec27..d8c52cbde04c 100644 --- a/llvm/lib/Target/RISCV/RISCVCallingConv.td +++ b/llvm/lib/Target/RISCV/RISCVCallingConv.td @@ -93,3 +93,7 @@ def CSR_XLEN_F32_V_Interrupt_RVE: CalleeSavedRegs<(sub CSR_XLEN_F32_V_Interrupt, // Same as CSR_XLEN_F64_V_Interrupt, but excluding X16-X31. def CSR_XLEN_F64_V_Interrupt_RVE: CalleeSavedRegs<(sub CSR_XLEN_F64_V_Interrupt, (sequence "X%u", 16, 31))>; + +def CSR_RT_MostRegs : CalleeSavedRegs<(sub CSR_Interrupt, X6, X7, X28)>; +def CSR_RT_MostRegs_RVE : CalleeSavedRegs<(sub CSR_RT_MostRegs, + (sequence "X%u", 16, 31))>; diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp index a796c910bd44..c4f41b8d8e4d 100644 --- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -738,7 +738,8 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB, MachineFunction &MF, uint64_t Offset, uint64_t RealStackSize, bool EmitCFI, bool NeedProbe, uint64_t ProbeSize, - bool DynAllocation) const { + bool DynAllocation, + MachineInstr::MIFlag Flag) const { DebugLoc DL; const RISCVRegisterInfo *RI = STI.getRegisterInfo(); const RISCVInstrInfo *TII = STI.getInstrInfo(); @@ -748,7 +749,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB, // Simply allocate the stack if it's not big enough to require a probe. if (!NeedProbe || Offset <= ProbeSize) { RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-Offset), - MachineInstr::FrameSetup, getStackAlign()); + Flag, getStackAlign()); if (EmitCFI) CFIBuilder.buildDefCFAOffset(RealStackSize); @@ -759,7 +760,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB, .addReg(RISCV::X0) .addReg(SPReg) .addImm(0) - .setMIFlags(MachineInstr::FrameSetup); + .setMIFlags(Flag); } return; @@ -767,30 +768,30 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB, // Unroll the probe loop depending on the number of iterations. if (Offset < ProbeSize * 5) { + uint64_t CFAAdjust = RealStackSize - Offset; + uint64_t CurrentOffset = 0; while (CurrentOffset + ProbeSize <= Offset) { RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, - StackOffset::getFixed(-ProbeSize), MachineInstr::FrameSetup, - getStackAlign()); + StackOffset::getFixed(-ProbeSize), Flag, getStackAlign()); // s[d|w] zero, 0(sp) BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW)) .addReg(RISCV::X0) .addReg(SPReg) .addImm(0) - .setMIFlags(MachineInstr::FrameSetup); + .setMIFlags(Flag); CurrentOffset += ProbeSize; if (EmitCFI) - CFIBuilder.buildDefCFAOffset(CurrentOffset); + CFIBuilder.buildDefCFAOffset(CurrentOffset + CFAAdjust); } uint64_t Residual = Offset - CurrentOffset; if (Residual) { RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, - StackOffset::getFixed(-Residual), MachineInstr::FrameSetup, - getStackAlign()); + StackOffset::getFixed(-Residual), Flag, getStackAlign()); if (EmitCFI) - CFIBuilder.buildDefCFAOffset(Offset); + CFIBuilder.buildDefCFAOffset(RealStackSize); if (DynAllocation) { // s[d|w] zero, 0(sp) @@ -798,7 +799,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB, .addReg(RISCV::X0) .addReg(SPReg) .addImm(0) - .setMIFlags(MachineInstr::FrameSetup); + .setMIFlags(Flag); } } @@ -812,8 +813,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB, Register TargetReg = RISCV::X6; // SUB TargetReg, SP, RoundedSize RI->adjustReg(MBB, MBBI, DL, TargetReg, SPReg, - StackOffset::getFixed(-RoundedSize), MachineInstr::FrameSetup, - getStackAlign()); + StackOffset::getFixed(-RoundedSize), Flag, getStackAlign()); if (EmitCFI) { // Set the CFA register to TargetReg. @@ -830,14 +830,14 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB, if (Residual) { RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-Residual), - MachineInstr::FrameSetup, getStackAlign()); + Flag, getStackAlign()); if (DynAllocation) { // s[d|w] zero, 0(sp) BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW)) .addReg(RISCV::X0) .addReg(SPReg) .addImm(0) - .setMIFlags(MachineInstr::FrameSetup); + .setMIFlags(Flag); } } @@ -1034,7 +1034,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF, MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation(); if (StackSize != 0) allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, /*EmitCFI=*/true, - NeedProbe, ProbeSize, DynAllocation); + NeedProbe, ProbeSize, DynAllocation, + MachineInstr::FrameSetup); // Save SiFive CLIC CSRs into Stack emitSiFiveCLICPreemptibleSaves(MF, MBB, MBBI, DL); @@ -1082,7 +1083,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF, allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount, getStackSizeWithRVVPadding(MF), !hasFP(MF), NeedProbe, - ProbeSize, DynAllocation); + ProbeSize, DynAllocation, MachineInstr::FrameSetup); } if (RVVStackSize) { @@ -1814,7 +1815,8 @@ MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr( bool DynAllocation = MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation(); allocateStack(MBB, MI, MF, -Amount, -Amount, !hasFP(MF), - /*NeedProbe=*/true, ProbeSize, DynAllocation); + /*NeedProbe=*/true, ProbeSize, DynAllocation, + MachineInstr::NoFlags); } else { const RISCVRegisterInfo &RI = *STI.getRegisterInfo(); RI.adjustReg(MBB, MI, DL, SPReg, SPReg, StackOffset::getFixed(Amount), diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.h b/llvm/lib/Target/RISCV/RISCVFrameLowering.h index d013755ce58a..6af63a4885f3 100644 --- a/llvm/lib/Target/RISCV/RISCVFrameLowering.h +++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.h @@ -81,7 +81,8 @@ public: void allocateStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineFunction &MF, uint64_t Offset, uint64_t RealStackSize, bool EmitCFI, bool NeedProbe, - uint64_t ProbeSize, bool DynAllocation) const; + uint64_t ProbeSize, bool DynAllocation, + MachineInstr::MIFlag Flag) const; protected: const RISCVSubtarget &STI; diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 666c76b21e63..186191abe12a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -689,10 +689,16 @@ bool RISCVDAGToDAGISel::trySignedBitfieldInsertInMask(SDNode *Node) { if (!isShiftedMask_32(C1) || isInt<12>(C1)) return false; + // INSBI will clobber the input register in N0. Bail out if we need a copy to + // preserve this value. + SDValue N0 = Node->getOperand(0); + if (!N0.hasOneUse()) + return false; + // If C1 is a shifted mask (but can't be formed as an ORI), // use a bitfield insert of -1. // Transform (or x, C1) - // -> (qc.insbi x, width, shift) + // -> (qc.insbi x, -1, width, shift) const unsigned Leading = llvm::countl_zero((uint32_t)C1); const unsigned Trailing = llvm::countr_zero((uint32_t)C1); const unsigned Width = 32 - Leading - Trailing; @@ -705,7 +711,7 @@ bool RISCVDAGToDAGISel::trySignedBitfieldInsertInMask(SDNode *Node) { SDLoc DL(Node); MVT VT = Node->getSimpleValueType(0); - SDValue Ops[] = {CurDAG->getSignedTargetConstant(-1, DL, VT), + SDValue Ops[] = {N0, CurDAG->getSignedTargetConstant(-1, DL, VT), CurDAG->getTargetConstant(Width, DL, VT), CurDAG->getTargetConstant(Trailing, DL, VT)}; SDNode *BitIns = CurDAG->getMachineNode(RISCV::QC_INSBI, DL, VT, Ops); @@ -2936,8 +2942,8 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base, /// Similar to SelectAddrRegImm, except that the offset is restricted to uimm9. bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base, SDValue &Offset) { - if (SelectAddrFrameIndex(Addr, Base, Offset)) - return true; + // FIXME: Support FrameIndex. Need to teach eliminateFrameIndex that only + // a 9-bit immediate can be folded. SDLoc DL(Addr); MVT VT = Addr.getSimpleValueType(); @@ -2947,8 +2953,8 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base, if (isUInt<9>(CVal)) { Base = Addr.getOperand(0); - if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base)) - Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT); + // FIXME: Support FrameIndex. Need to teach eliminateFrameIndex that only + // a 9-bit immediate can be folded. Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT); return true; } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 9cbc364afc21..567ef2ff19f0 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -16013,7 +16013,7 @@ static SDValue expandMul(SDNode *N, SelectionDAG &DAG, uint64_t MulAmt = CNode->getZExtValue(); // Don't do this if the Xqciac extension is enabled and the MulAmt in simm12. - if (Subtarget.hasVendorXqciac() && isInt<12>(MulAmt)) + if (Subtarget.hasVendorXqciac() && isInt<12>(CNode->getSExtValue())) return SDValue(); const bool HasShlAdd = Subtarget.hasStdExtZba() || @@ -16936,18 +16936,9 @@ struct NodeExtensionHelper { case RISCVISD::VWSUBU_W_VL: case RISCVISD::VFWADD_W_VL: case RISCVISD::VFWSUB_W_VL: - if (OperandIdx == 1) { - SupportsZExt = - Opc == RISCVISD::VWADDU_W_VL || Opc == RISCVISD::VWSUBU_W_VL; - SupportsSExt = - Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWSUB_W_VL; - SupportsFPExt = - Opc == RISCVISD::VFWADD_W_VL || Opc == RISCVISD::VFWSUB_W_VL; - // There's no existing extension here, so we don't have to worry about - // making sure it gets removed. - EnforceOneUse = false; + // Operand 1 can't be changed. + if (OperandIdx == 1) break; - } [[fallthrough]]; default: fillUpExtensionSupport(Root, DAG, Subtarget); @@ -16985,20 +16976,20 @@ struct NodeExtensionHelper { case RISCVISD::ADD_VL: case RISCVISD::MUL_VL: case RISCVISD::OR_VL: - case RISCVISD::VWADD_W_VL: - case RISCVISD::VWADDU_W_VL: case RISCVISD::FADD_VL: case RISCVISD::FMUL_VL: - case RISCVISD::VFWADD_W_VL: case RISCVISD::VFMADD_VL: case RISCVISD::VFNMSUB_VL: case RISCVISD::VFNMADD_VL: case RISCVISD::VFMSUB_VL: return true; + case RISCVISD::VWADD_W_VL: + case RISCVISD::VWADDU_W_VL: case ISD::SUB: case RISCVISD::SUB_VL: case RISCVISD::VWSUB_W_VL: case RISCVISD::VWSUBU_W_VL: + case RISCVISD::VFWADD_W_VL: case RISCVISD::FSUB_VL: case RISCVISD::VFWSUB_W_VL: case ISD::SHL: @@ -17117,6 +17108,30 @@ canFoldToVWWithSameExtension(SDNode *Root, const NodeExtensionHelper &LHS, Subtarget); } +/// Check if \p Root follows a pattern Root(zext(LHS), zext(RHS)) +/// +/// \returns std::nullopt if the pattern doesn't match or a CombineResult that +/// can be used to apply the pattern. +static std::optional<CombineResult> +canFoldToVWWithSameExtZEXT(SDNode *Root, const NodeExtensionHelper &LHS, + const NodeExtensionHelper &RHS, SelectionDAG &DAG, + const RISCVSubtarget &Subtarget) { + return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG, + Subtarget); +} + +/// Check if \p Root follows a pattern Root(bf16ext(LHS), bf16ext(RHS)) +/// +/// \returns std::nullopt if the pattern doesn't match or a CombineResult that +/// can be used to apply the pattern. +static std::optional<CombineResult> +canFoldToVWWithSameExtBF16(SDNode *Root, const NodeExtensionHelper &LHS, + const NodeExtensionHelper &RHS, SelectionDAG &DAG, + const RISCVSubtarget &Subtarget) { + return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::BF16Ext, DAG, + Subtarget); +} + /// Check if \p Root follows a pattern Root(LHS, ext(RHS)) /// /// \returns std::nullopt if the pattern doesn't match or a CombineResult that @@ -17145,7 +17160,7 @@ canFoldToVW_W(SDNode *Root, const NodeExtensionHelper &LHS, return std::nullopt; } -/// Check if \p Root follows a pattern Root(sext(LHS), sext(RHS)) +/// Check if \p Root follows a pattern Root(sext(LHS), RHS) /// /// \returns std::nullopt if the pattern doesn't match or a CombineResult that /// can be used to apply the pattern. @@ -17153,11 +17168,14 @@ static std::optional<CombineResult> canFoldToVWWithSEXT(SDNode *Root, const NodeExtensionHelper &LHS, const NodeExtensionHelper &RHS, SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { - return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG, - Subtarget); + if (LHS.SupportsSExt) + return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->getOpcode()), + Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS, + /*RHSExt=*/std::nullopt); + return std::nullopt; } -/// Check if \p Root follows a pattern Root(zext(LHS), zext(RHS)) +/// Check if \p Root follows a pattern Root(zext(LHS), RHS) /// /// \returns std::nullopt if the pattern doesn't match or a CombineResult that /// can be used to apply the pattern. @@ -17165,11 +17183,14 @@ static std::optional<CombineResult> canFoldToVWWithZEXT(SDNode *Root, const NodeExtensionHelper &LHS, const NodeExtensionHelper &RHS, SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { - return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG, - Subtarget); + if (LHS.SupportsZExt) + return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->getOpcode()), + Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS, + /*RHSExt=*/std::nullopt); + return std::nullopt; } -/// Check if \p Root follows a pattern Root(fpext(LHS), fpext(RHS)) +/// Check if \p Root follows a pattern Root(fpext(LHS), RHS) /// /// \returns std::nullopt if the pattern doesn't match or a CombineResult that /// can be used to apply the pattern. @@ -17177,20 +17198,11 @@ static std::optional<CombineResult> canFoldToVWWithFPEXT(SDNode *Root, const NodeExtensionHelper &LHS, const NodeExtensionHelper &RHS, SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { - return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG, - Subtarget); -} - -/// Check if \p Root follows a pattern Root(bf16ext(LHS), bf16ext(RHS)) -/// -/// \returns std::nullopt if the pattern doesn't match or a CombineResult that -/// can be used to apply the pattern. -static std::optional<CombineResult> -canFoldToVWWithBF16EXT(SDNode *Root, const NodeExtensionHelper &LHS, - const NodeExtensionHelper &RHS, SelectionDAG &DAG, - const RISCVSubtarget &Subtarget) { - return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::BF16Ext, DAG, - Subtarget); + if (LHS.SupportsFPExt) + return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->getOpcode()), + Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS, + /*RHSExt=*/std::nullopt); + return std::nullopt; } /// Check if \p Root follows a pattern Root(sext(LHS), zext(RHS)) @@ -17233,7 +17245,7 @@ NodeExtensionHelper::getSupportedFoldings(const SDNode *Root) { case RISCVISD::VFNMSUB_VL: Strategies.push_back(canFoldToVWWithSameExtension); if (Root->getOpcode() == RISCVISD::VFMADD_VL) - Strategies.push_back(canFoldToVWWithBF16EXT); + Strategies.push_back(canFoldToVWWithSameExtBF16); break; case ISD::MUL: case RISCVISD::MUL_VL: @@ -17245,7 +17257,7 @@ NodeExtensionHelper::getSupportedFoldings(const SDNode *Root) { case ISD::SHL: case RISCVISD::SHL_VL: // shl -> vwsll - Strategies.push_back(canFoldToVWWithZEXT); + Strategies.push_back(canFoldToVWWithSameExtZEXT); break; case RISCVISD::VWADD_W_VL: case RISCVISD::VWSUB_W_VL: @@ -22224,6 +22236,7 @@ SDValue RISCVTargetLowering::LowerFormalArguments( case CallingConv::C: case CallingConv::Fast: case CallingConv::SPIR_KERNEL: + case CallingConv::PreserveMost: case CallingConv::GRAAL: case CallingConv::RISCV_VectorCall: #define CC_VLS_CASE(ABI_VLEN) case CallingConv::RISCV_VLSCall_##ABI_VLEN: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td index 26bb1e8d1785..f39130090def 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td @@ -845,10 +845,11 @@ let Predicates = [HasVendorXqcibi, IsRV32] in { let Predicates = [HasVendorXqcibm, IsRV32] in { let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { def QC_INSBRI : QCIRVInstRI<0b1, simm11, "qc.insbri">; - def QC_INSBI : RVInstIBase<0b001, OPC_CUSTOM_0, (outs GPRNoX0:$rd), - (ins simm5:$imm5, uimm5_plus1:$width, + def QC_INSBI : RVInstIBase<0b001, OPC_CUSTOM_0, (outs GPRNoX0:$rd_wb), + (ins GPRNoX0:$rd, simm5:$imm5, uimm5_plus1:$width, uimm5:$shamt), "qc.insbi", "$rd, $imm5, $width, $shamt"> { + let Constraints = "$rd = $rd_wb"; bits<5> imm5; bits<5> shamt; bits<5> width; @@ -1376,9 +1377,9 @@ let Predicates = [HasVendorXqciac, IsRV32] in { def : Pat<(i32 (add GPRNoX0:$rd, (mul GPRNoX0:$rs1, simm12:$imm12))), (QC_MULIADD GPRNoX0:$rd, GPRNoX0:$rs1, simm12:$imm12)>; def : Pat<(i32 (add_like_non_imm12 (shl GPRNoX0:$rs1, uimm5gt3:$imm), GPRNoX0:$rs2)), - (QC_SHLADD GPRNoX0:$rs2, GPRNoX0:$rs1, uimm5gt3:$imm)>; + (QC_SHLADD GPRNoX0:$rs1, GPRNoX0:$rs2, uimm5gt3:$imm)>; def : Pat<(i32 (riscv_shl_add GPRNoX0:$rs1, uimm5gt3:$imm, GPRNoX0:$rs2)), - (QC_SHLADD GPRNoX0:$rs2, GPRNoX0:$rs1, uimm5gt3:$imm)>; + (QC_SHLADD GPRNoX0:$rs1, GPRNoX0:$rs2, uimm5gt3:$imm)>; } // Predicates = [HasVendorXqciac, IsRV32] /// Simple arithmetic operations diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td index 4147c97a7a23..92bc3ee8bdac 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td @@ -1130,13 +1130,13 @@ let Predicates = [HasStdExtZvkned] in { let Predicates = [HasStdExtZvknha] in { defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ch", "PseudoVSHA2CH", I32IntegerVectors>; - defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2cl", "PseudoVSHA2CH", I32IntegerVectors>; + defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2cl", "PseudoVSHA2CL", I32IntegerVectors>; defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ms", "PseudoVSHA2MS", I32IntegerVectors, isSEWAware=true>; } // Predicates = [HasStdExtZvknha] let Predicates = [HasStdExtZvknhb] in { defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ch", "PseudoVSHA2CH", I32I64IntegerVectors>; - defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2cl", "PseudoVSHA2CH", I32I64IntegerVectors>; + defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2cl", "PseudoVSHA2CL", I32I64IntegerVectors>; defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ms", "PseudoVSHA2MS", I32I64IntegerVectors, isSEWAware=true>; } // Predicates = [HasStdExtZvknhb] diff --git a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp index 7a2541a652b5..0d37db0138e4 100644 --- a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp +++ b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp @@ -137,6 +137,11 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I, NextI = next_nodbg(NextI, E); DebugLoc DL = I->getDebugLoc(); + // Make a copy so we can update the kill flag in the MoveFromAToS case. The + // copied operand needs to be scoped outside the if since we make a pointer + // to it. + MachineOperand PairedSource = *PairedRegs.Source; + // The order of S-reg depends on which instruction holds A0, instead of // the order of register pair. // e,g. @@ -147,8 +152,15 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I, // mv a1, s1 => cm.mva01s s2,s1 bool StartWithX10 = ARegInFirstPair == RISCV::X10; if (isMoveFromAToS(Opcode)) { - Sreg1 = StartWithX10 ? FirstPair.Source : PairedRegs.Source; - Sreg2 = StartWithX10 ? PairedRegs.Source : FirstPair.Source; + // We are moving one of the copies earlier so its kill flag may become + // invalid. Clear the copied kill flag if there are any reads of the + // register between the new location and the old location. + for (auto It = std::next(I); It != Paired && PairedSource.isKill(); ++It) + if (It->readsRegister(PairedSource.getReg(), TRI)) + PairedSource.setIsKill(false); + + Sreg1 = StartWithX10 ? FirstPair.Source : &PairedSource; + Sreg2 = StartWithX10 ? &PairedSource : FirstPair.Source; } else { Sreg1 = StartWithX10 ? FirstPair.Destination : PairedRegs.Destination; Sreg2 = StartWithX10 ? PairedRegs.Destination : FirstPair.Destination; diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 540412366026..214536d7f3a7 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -68,6 +68,9 @@ RISCVRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { auto &Subtarget = MF->getSubtarget<RISCVSubtarget>(); if (MF->getFunction().getCallingConv() == CallingConv::GHC) return CSR_NoRegs_SaveList; + if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost) + return Subtarget.hasStdExtE() ? CSR_RT_MostRegs_RVE_SaveList + : CSR_RT_MostRegs_SaveList; if (MF->getFunction().hasFnAttribute("interrupt")) { if (Subtarget.hasVInstructions()) { if (Subtarget.hasStdExtD()) @@ -811,7 +814,13 @@ RISCVRegisterInfo::getCallPreservedMask(const MachineFunction & MF, if (CC == CallingConv::GHC) return CSR_NoRegs_RegMask; - switch (Subtarget.getTargetABI()) { + RISCVABI::ABI ABI = Subtarget.getTargetABI(); + if (CC == CallingConv::PreserveMost) { + if (ABI == RISCVABI::ABI_ILP32E || ABI == RISCVABI::ABI_LP64E) + return CSR_RT_MostRegs_RVE_RegMask; + return CSR_RT_MostRegs_RegMask; + } + switch (ABI) { default: llvm_unreachable("Unrecognized ABI"); case RISCVABI::ABI_ILP32E: diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 56ead92187b0..70cfb4b836cc 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1567,6 +1567,7 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, // scalarized if the legalized Src and Dst are not equal sized. const DataLayout &DL = this->getDataLayout(); if (!SrcLT.second.isVector() || !DstLT.second.isVector() || + !SrcLT.first.isValid() || !DstLT.first.isValid() || !TypeSize::isKnownLE(DL.getTypeSizeInBits(Src), SrcLT.second.getSizeInBits()) || !TypeSize::isKnownLE(DL.getTypeSizeInBits(Dst), diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp index 84ef53985484..c1cc19b503de 100644 --- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp +++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp @@ -434,6 +434,15 @@ bool RISCVVectorPeephole::convertSameMaskVMergeToVMv(MachineInstr &MI) { if (!isKnownSameDefs(TrueMask.getReg(), MIMask.getReg())) return false; + // Masked off lanes past TrueVL will come from False, and converting to vmv + // will lose these lanes unless MIVL <= TrueVL. + // TODO: We could relax this for False == Passthru and True policy == TU + const MachineOperand &MIVL = MI.getOperand(RISCVII::getVLOpNum(MI.getDesc())); + const MachineOperand &TrueVL = + True->getOperand(RISCVII::getVLOpNum(True->getDesc())); + if (!RISCV::isVLKnownLE(MIVL, TrueVL)) + return false; + // True's passthru needs to be equivalent to False Register TruePassthruReg = True->getOperand(1).getReg(); Register FalseReg = MI.getOperand(2).getReg(); diff --git a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp index 6c19049a001c..024030d196ee 100644 --- a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp +++ b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp @@ -206,8 +206,8 @@ Filler::findDelayInstr(MachineBasicBlock &MBB, if (!done) --I; - // skip debug instruction - if (I->isDebugInstr()) + // Skip meta instructions. + if (I->isMetaInstruction()) continue; if (I->hasUnmodeledSideEffects() || I->isInlineAsm() || I->isPosition() || diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp index 9b434d87c267..57e7ff3a52ce 100644 --- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp +++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp @@ -115,7 +115,8 @@ static bool Analyze_CC_Sparc64_Full(bool IsReturn, unsigned &ValNo, MVT &ValVT, // Stack space is allocated for all arguments starting from [%fp+BIAS+128]. unsigned size = (LocVT == MVT::f128) ? 16 : 8; - Align alignment = (LocVT == MVT::f128) ? Align(16) : Align(8); + Align alignment = + (LocVT == MVT::f128 || ArgFlags.isSplit()) ? Align(16) : Align(8); unsigned Offset = State.AllocateStack(size, alignment); unsigned Reg = 0; @@ -1799,12 +1800,14 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM, setOperationAction(ISD::FCOS , MVT::f64, Expand); setOperationAction(ISD::FSINCOS, MVT::f64, Expand); setOperationAction(ISD::FREM , MVT::f64, Expand); - setOperationAction(ISD::FMA , MVT::f64, Expand); + setOperationAction(ISD::FMA, MVT::f64, + Subtarget->isUA2007() ? Legal : Expand); setOperationAction(ISD::FSIN , MVT::f32, Expand); setOperationAction(ISD::FCOS , MVT::f32, Expand); setOperationAction(ISD::FSINCOS, MVT::f32, Expand); setOperationAction(ISD::FREM , MVT::f32, Expand); - setOperationAction(ISD::FMA, MVT::f32, Expand); + setOperationAction(ISD::FMA, MVT::f32, + Subtarget->isUA2007() ? Legal : Expand); setOperationAction(ISD::ROTL , MVT::i32, Expand); setOperationAction(ISD::ROTR , MVT::i32, Expand); setOperationAction(ISD::BSWAP, MVT::i32, Expand); @@ -2201,7 +2204,7 @@ SDValue SparcTargetLowering::LowerGlobalTLSAddress(SDValue Op, SDValue Chain = DAG.getEntryNode(); SDValue InGlue; - Chain = DAG.getCALLSEQ_START(Chain, 1, 0, DL); + Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL); Chain = DAG.getCopyToReg(Chain, DL, SP::O0, Argument, InGlue); InGlue = Chain.getValue(1); SDValue Callee = DAG.getTargetExternalSymbol("__tls_get_addr", PtrVT); @@ -2219,7 +2222,7 @@ SDValue SparcTargetLowering::LowerGlobalTLSAddress(SDValue Op, InGlue}; Chain = DAG.getNode(SPISD::TLS_CALL, DL, NodeTys, Ops); InGlue = Chain.getValue(1); - Chain = DAG.getCALLSEQ_END(Chain, 1, 0, InGlue, DL); + Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL); InGlue = Chain.getValue(1); SDValue Ret = DAG.getCopyFromReg(Chain, DL, SP::O0, PtrVT, InGlue); @@ -3550,6 +3553,11 @@ bool SparcTargetLowering::isCheapToSpeculateCttz(Type *Ty) const { return isCheapToSpeculateCtlz(Ty); } +bool SparcTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, + EVT VT) const { + return Subtarget->isUA2007() && !Subtarget->useSoftFloat(); +} + // Override to disable global variable loading on Linux. void SparcTargetLowering::insertSSPDeclarations(Module &M) const { if (!Subtarget->isTargetLinux()) diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.h b/llvm/lib/Target/Sparc/SparcISelLowering.h index 0d220f8c3d32..4017beb88ff3 100644 --- a/llvm/lib/Target/Sparc/SparcISelLowering.h +++ b/llvm/lib/Target/Sparc/SparcISelLowering.h @@ -177,6 +177,11 @@ namespace llvm { bool isCheapToSpeculateCttz(Type *Ty) const override; + bool enableAggressiveFMAFusion(EVT VT) const override { return true; }; + + bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, + EVT VT) const override; + bool shouldInsertFencesForAtomic(const Instruction *I) const override { // FIXME: We insert fences for each atomics and generate // sub-optimal code for PSO/TSO. (Approximately nobody uses any diff --git a/llvm/lib/Target/Sparc/SparcInstrUAOSA.td b/llvm/lib/Target/Sparc/SparcInstrUAOSA.td index 3a30e552e6db..ffd4423137e3 100644 --- a/llvm/lib/Target/Sparc/SparcInstrUAOSA.td +++ b/llvm/lib/Target/Sparc/SparcInstrUAOSA.td @@ -66,3 +66,15 @@ defm CXBCOND : F2_56<"cxb", 1>; def FPMADDX : FourOp<"fpmaddx", 0b110111, 0b0000, DFPRegs>; def FPMADDXHI : FourOp<"fpmaddxhi", 0b110111, 0b0100, DFPRegs>; } // Predicates = [HasOSA2011] + +// UA2007 instruction patterns. +let Predicates = [HasUA2007] in { +def : Pat<(f32 (any_fma f32:$rs1, f32:$rs2, f32:$add)), (FMADDS $rs1, $rs2, $add)>; +def : Pat<(f64 (any_fma f64:$rs1, f64:$rs2, f64:$add)), (FMADDD $rs1, $rs2, $add)>; +def : Pat<(f32 (any_fma f32:$rs1, f32:$rs2, (fneg f32:$sub))), (FMSUBS $rs1, $rs2, $sub)>; +def : Pat<(f64 (any_fma f64:$rs1, f64:$rs2, (fneg f64:$sub))), (FMSUBD $rs1, $rs2, $sub)>; +def : Pat<(f32 (fneg (any_fma f32:$rs1, f32:$rs2, f32:$add))), (FNMADDS $rs1, $rs2, $add)>; +def : Pat<(f64 (fneg (any_fma f64:$rs1, f64:$rs2, f64:$add))), (FNMADDD $rs1, $rs2, $add)>; +def : Pat<(f32 (fneg (any_fma f32:$rs1, f32:$rs2, (fneg f32:$sub)))), (FNMSUBS $rs1, $rs2, $sub)>; +def : Pat<(f64 (fneg (any_fma f64:$rs1, f64:$rs2, (fneg f64:$sub)))), (FNMSUBD $rs1, $rs2, $sub)>; +} // Predicates = [HasUA2007] diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp index f32c9bd2bdea..2611c291abaa 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -436,20 +436,6 @@ bool SystemZTTIImpl::isLSRCostLess( C2.ScaleCost, C2.SetupCost); } -bool SystemZTTIImpl::areInlineCompatible(const Function *Caller, - const Function *Callee) const { - const TargetMachine &TM = getTLI()->getTargetMachine(); - - const FeatureBitset &CallerBits = - TM.getSubtargetImpl(*Caller)->getFeatureBits(); - const FeatureBitset &CalleeBits = - TM.getSubtargetImpl(*Callee)->getFeatureBits(); - - // Support only equal feature bitsets. Restriction should be relaxed in the - // future to allow inlining when callee's bits are subset of the caller's. - return CallerBits == CalleeBits; -} - unsigned SystemZTTIImpl::getNumberOfRegisters(unsigned ClassID) const { bool Vector = (ClassID == 1); if (!Vector) diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h index dc5736e8af00..fc681dec1859 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h @@ -65,9 +65,6 @@ public: bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, const TargetTransformInfo::LSRCost &C2) const override; - bool areInlineCompatible(const Function *Caller, - const Function *Callee) const override; - /// @} /// \name Vector TTI Implementations diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp index 2662241ef849..e6486e247209 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp @@ -256,9 +256,17 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) { // Precompute the set of registers that are unused, so that we can insert // drops to their defs. + // And unstackify any stackified registers that don't have any uses, so that + // they can be dropped later. This can happen when transformations after + // RegStackify remove instructions using stackified registers. BitVector UseEmpty(MRI.getNumVirtRegs()); - for (unsigned I = 0, E = MRI.getNumVirtRegs(); I < E; ++I) - UseEmpty[I] = MRI.use_empty(Register::index2VirtReg(I)); + for (unsigned I = 0, E = MRI.getNumVirtRegs(); I < E; ++I) { + Register Reg = Register::index2VirtReg(I); + if (MRI.use_empty(Reg)) { + UseEmpty[I] = true; + MFI.unstackifyVReg(Reg); + } + } // Visit each instruction in the function. for (MachineBasicBlock &MBB : MF) { diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp index b642c1cfe383..8213e512f45e 100644 --- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -1042,8 +1042,8 @@ private: } PrevState = CurrState; } - void onRParen() { - PrevState = State; + bool onRParen(StringRef &ErrMsg) { + IntelExprState CurrState = State; switch (State) { default: State = IES_ERROR; @@ -1054,9 +1054,27 @@ private: case IES_RBRAC: case IES_RPAREN: State = IES_RPAREN; + // In the case of a multiply, onRegister has already set IndexReg + // directly, with appropriate scale. + // Otherwise if we just saw a register it has only been stored in + // TmpReg, so we need to store it into the state machine. + if (CurrState == IES_REGISTER && PrevState != IES_MULTIPLY) { + // If we already have a BaseReg, then assume this is the IndexReg with + // no explicit scale. + if (!BaseReg) { + BaseReg = TmpReg; + } else { + if (IndexReg) + return regsUseUpError(ErrMsg); + IndexReg = TmpReg; + Scale = 0; + } + } IC.pushOperator(IC_RPAREN); break; } + PrevState = CurrState; + return false; } bool onOffset(const MCExpr *Val, SMLoc OffsetLoc, StringRef ID, const InlineAsmIdentifierInfo &IDInfo, @@ -2172,7 +2190,11 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) { } break; case AsmToken::LParen: SM.onLParen(); break; - case AsmToken::RParen: SM.onRParen(); break; + case AsmToken::RParen: + if (SM.onRParen(ErrMsg)) { + return Error(Tok.getLoc(), ErrMsg); + } + break; } if (SM.hadError()) return Error(Tok.getLoc(), "unknown token in expression"); diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp index 0b4c63f7a81f..5d5a70589324 100644 --- a/llvm/lib/Target/X86/X86CallingConv.cpp +++ b/llvm/lib/Target/X86/X86CallingConv.cpp @@ -374,5 +374,36 @@ static bool CC_X86_64_I128(unsigned &ValNo, MVT &ValVT, MVT &LocVT, return true; } +/// Special handling for i128 and fp128: on x86-32, i128 and fp128 get legalized +/// as four i32s, but fp128 must be passed on the stack with 16-byte alignment. +/// Technically only fp128 has a specified ABI, but it makes sense to handle +/// i128 the same until we hear differently. +static bool CC_X86_32_I128_FP128(unsigned &ValNo, MVT &ValVT, MVT &LocVT, + CCValAssign::LocInfo &LocInfo, + ISD::ArgFlagsTy &ArgFlags, CCState &State) { + assert(ValVT == MVT::i32 && "Should have i32 parts"); + SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs(); + PendingMembers.push_back( + CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); + + if (!ArgFlags.isInConsecutiveRegsLast()) + return true; + + assert(PendingMembers.size() == 4 && "Should have four parts"); + + int64_t Offset = State.AllocateStack(16, Align(16)); + PendingMembers[0].convertToMem(Offset); + PendingMembers[1].convertToMem(Offset + 4); + PendingMembers[2].convertToMem(Offset + 8); + PendingMembers[3].convertToMem(Offset + 12); + + State.addLoc(PendingMembers[0]); + State.addLoc(PendingMembers[1]); + State.addLoc(PendingMembers[2]); + State.addLoc(PendingMembers[3]); + PendingMembers.clear(); + return true; +} + // Provides entry points of CC_X86 and RetCC_X86. #include "X86GenCallingConv.inc" diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td index 823e0caa0226..f020e0b55141 100644 --- a/llvm/lib/Target/X86/X86CallingConv.td +++ b/llvm/lib/Target/X86/X86CallingConv.td @@ -859,6 +859,11 @@ def CC_X86_32_C : CallingConv<[ // The 'nest' parameter, if any, is passed in ECX. CCIfNest<CCAssignToReg<[ECX]>>, + // i128 and fp128 need to be passed on the stack with a higher alignment than + // their legal types. Handle this with a custom function. + CCIfType<[i32], + CCIfConsecutiveRegs<CCCustom<"CC_X86_32_I128_FP128">>>, + // On swifttailcc pass swiftself in ECX. CCIfCC<"CallingConv::SwiftTail", CCIfSwiftSelf<CCIfType<[i32], CCAssignToReg<[ECX]>>>>, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 5e35d5630d66..c91bf269076e 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -4997,9 +4997,12 @@ static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits, EVT VT = Op.getValueType(); unsigned SizeInBits = VT.getSizeInBits(); - assert((SizeInBits % EltSizeInBits) == 0 && "Can't split constant!"); unsigned NumElts = SizeInBits / EltSizeInBits; + // Can't split constant. + if ((SizeInBits % EltSizeInBits) != 0) + return false; + // Bitcast a source array of element bits to the target size. auto CastBitData = [&](APInt &UndefSrcElts, ArrayRef<APInt> SrcEltBits) { unsigned NumSrcElts = UndefSrcElts.getBitWidth(); @@ -15400,18 +15403,18 @@ static SDValue lowerShuffleAsLanePermuteAndPermute( return SDValue(); } - // Avoid returning the same shuffle operation. For example, - // t7: v16i16 = vector_shuffle<8,9,10,11,4,5,6,7,0,1,2,3,12,13,14,15> t5, - // undef:v16i16 - if (CrossLaneMask == Mask || InLaneMask == Mask) - return SDValue(); - // Simplify CrossLaneMask based on the actual demanded elements. if (V1.hasOneUse()) for (int i = 0; i != NumElts; ++i) if (!DemandedCrossLane[i]) CrossLaneMask[i] = SM_SentinelUndef; + // Avoid returning the same shuffle operation. For example, + // t7: v16i16 = vector_shuffle<8,9,10,11,4,5,6,7,0,1,2,3,12,13,14,15> t5, + // undef:v16i16 + if (CrossLaneMask == Mask || InLaneMask == Mask) + return SDValue(); + SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask); return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT), InLaneMask); @@ -26233,10 +26236,9 @@ static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - - if (auto *MaskConst = dyn_cast<ConstantSDNode>(Mask)) - if (MaskConst->getZExtValue() & 0x1) - return Op; + auto *MaskConst = dyn_cast<ConstantSDNode>(Mask); + if (MaskConst && (MaskConst->getZExtValue() & 0x1)) + return Op; MVT VT = Op.getSimpleValueType(); SDLoc dl(Op); @@ -26252,6 +26254,17 @@ static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask, if (PreservedSrc.isUndef()) PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); + + if (MaskConst) { + assert((MaskConst->getZExtValue() & 0x1) == 0 && "Expected false mask"); + // Discard op and blend passthrough with scalar op src/dst. + SmallVector<int, 16> ShuffleMask(VT.getVectorNumElements()); + std::iota(ShuffleMask.begin(), ShuffleMask.end(), 0); + ShuffleMask[0] = VT.getVectorNumElements(); + return DAG.getVectorShuffle(VT, dl, Op.getOperand(0), PreservedSrc, + ShuffleMask); + } + return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc); } @@ -44175,8 +44188,12 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( } // Conversions. // TODO: Add more CVT opcodes when we have test coverage. - case X86ISD::CVTTP2SI: case X86ISD::CVTTP2UI: { + if (!Subtarget.hasVLX()) + break; + [[fallthrough]]; + } + case X86ISD::CVTTP2SI: { if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f16 && !Subtarget.hasVLX()) break; @@ -51758,6 +51775,8 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, SDValue X, Y; EVT CondVT = VT.changeVectorElementType(MVT::i1); if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(CondVT) && + (VT.is512BitVector() || Subtarget.hasVLX()) && + (VT.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) && sd_match(N, m_And(m_Value(X), m_OneUse(m_SExt(m_AllOf( m_Value(Y), m_SpecificVT(CondVT), @@ -55312,6 +55331,8 @@ static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG, SDValue Src = N0.getOperand(0); EVT SrcVT = Src.getValueType(); if (Src.getOpcode() == ISD::SETCC && SrcVT.getScalarType() == MVT::i1 && + (VT.is512BitVector() || Subtarget.hasVLX()) && + (VT.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) && TLI.isTypeLegal(SrcVT) && N0.hasOneUse() && Src.hasOneUse()) return DAG.getSelect(DL, VT, DAG.getNOT(DL, Src, SrcVT), N1, getZeroVector(VT, Subtarget, DAG, DL)); @@ -56191,7 +56212,13 @@ static SDValue combineAVX512SetCCToKMOV(EVT VT, SDValue Op0, ISD::CondCode CC, SDValue Masked = BroadcastOp; if (N != 0) { - APInt Mask = APInt::getLowBitsSet(BroadcastOpVT.getSizeInBits(), Len); + unsigned BroadcastOpBitWidth = BroadcastOpVT.getSizeInBits(); + unsigned NumDefinedElts = UndefElts.countTrailingZeros(); + + if (NumDefinedElts > BroadcastOpBitWidth) + return SDValue(); + + APInt Mask = APInt::getLowBitsSet(BroadcastOpBitWidth, NumDefinedElts); SDValue ShiftedValue = DAG.getNode(ISD::SRL, DL, BroadcastOpVT, BroadcastOp, DAG.getConstant(N, DL, BroadcastOpVT)); Masked = DAG.getNode(ISD::AND, DL, BroadcastOpVT, ShiftedValue, diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index 9ad355311527..b4639ac2577e 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -237,9 +237,18 @@ EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL, bool X86TargetLowering::functionArgumentNeedsConsecutiveRegisters( Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const { - // i128 split into i64 needs to be allocated to two consecutive registers, - // or spilled to the stack as a whole. - return Ty->isIntegerTy(128); + // On x86-64 i128 is split into two i64s and needs to be allocated to two + // consecutive registers, or spilled to the stack as a whole. On x86-32 i128 + // is split to four i32s and never actually passed in registers, but we use + // the consecutive register mark to match it in TableGen. + if (Ty->isIntegerTy(128)) + return true; + + // On x86-32, fp128 acts the same as i128. + if (Subtarget.is32Bit() && Ty->isFP128Ty()) + return true; + + return false; } /// Helper for getByValTypeAlignment to determine diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index abf365eedec3..9bf58dd3458c 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -10739,39 +10739,27 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB, if (!ST.hasSSE1()) return; - // PXOR is safe to use because it doesn't affect flags. - BuildMI(MBB, Iter, DL, get(X86::PXORrr), Reg) - .addReg(Reg, RegState::Undef) - .addReg(Reg, RegState::Undef); + BuildMI(MBB, Iter, DL, get(X86::V_SET0), Reg); } else if (X86::VR256RegClass.contains(Reg)) { // YMM# if (!ST.hasAVX()) return; - // VPXOR is safe to use because it doesn't affect flags. - BuildMI(MBB, Iter, DL, get(X86::VPXORrr), Reg) - .addReg(Reg, RegState::Undef) - .addReg(Reg, RegState::Undef); + BuildMI(MBB, Iter, DL, get(X86::AVX_SET0), Reg); } else if (X86::VR512RegClass.contains(Reg)) { // ZMM# if (!ST.hasAVX512()) return; - // VPXORY is safe to use because it doesn't affect flags. - BuildMI(MBB, Iter, DL, get(X86::VPXORYrr), Reg) - .addReg(Reg, RegState::Undef) - .addReg(Reg, RegState::Undef); + BuildMI(MBB, Iter, DL, get(X86::AVX512_512_SET0), Reg); } else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) || X86::VK4RegClass.contains(Reg) || X86::VK8RegClass.contains(Reg) || X86::VK16RegClass.contains(Reg)) { if (!ST.hasVLX()) return; - // KXOR is safe to use because it doesn't affect flags. - unsigned Op = ST.hasBWI() ? X86::KXORQkk : X86::KXORWkk; - BuildMI(MBB, Iter, DL, get(Op), Reg) - .addReg(Reg, RegState::Undef) - .addReg(Reg, RegState::Undef); + unsigned Op = ST.hasBWI() ? X86::KSET0Q : X86::KSET0W; + BuildMI(MBB, Iter, DL, get(Op), Reg); } } diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp index fd42fd2e010b..f847ddb46af7 100644 --- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp +++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp @@ -232,12 +232,6 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM, setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); - - setCondCodeAction(ISD::SETOGT, MVT::f32, Expand); - setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); - setCondCodeAction(ISD::SETONE, MVT::f32, Expand); - setCondCodeAction(ISD::SETUGE, MVT::f32, Expand); - setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); } else { setOperationAction(ISD::BITCAST, MVT::i32, Expand); setOperationAction(ISD::BITCAST, MVT::f32, Expand); @@ -877,6 +871,16 @@ static std::pair<unsigned, unsigned> getFPBranchKind(ISD::CondCode Cond) { return std::make_pair(Xtensa::BF, Xtensa::OLT_S); case ISD::SETGT: return std::make_pair(Xtensa::BF, Xtensa::OLE_S); + case ISD::SETOGT: + return std::make_pair(Xtensa::BF, Xtensa::ULE_S); + case ISD::SETOGE: + return std::make_pair(Xtensa::BF, Xtensa::ULT_S); + case ISD::SETONE: + return std::make_pair(Xtensa::BF, Xtensa::UEQ_S); + case ISD::SETUGT: + return std::make_pair(Xtensa::BF, Xtensa::OLE_S); + case ISD::SETUGE: + return std::make_pair(Xtensa::BF, Xtensa::OLT_S); default: llvm_unreachable("Invalid condition!"); } diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp index 59ae057cae79..ac93f748ce65 100644 --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -85,6 +85,9 @@ static Intrinsic::ID NonOverloadedCoroIntrinsics[] = { Intrinsic::coro_id_async, Intrinsic::coro_id_retcon, Intrinsic::coro_id_retcon_once, + Intrinsic::coro_noop, + Intrinsic::coro_prepare_async, + Intrinsic::coro_prepare_retcon, Intrinsic::coro_promise, Intrinsic::coro_resume, Intrinsic::coro_save, diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp index e276376f2158..0d631734ca96 100644 --- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -350,12 +350,20 @@ void splitAndWriteThinLTOBitcode( }); } + auto MustEmitToMergedModule = [](const GlobalValue *GV) { + // The __cfi_check definition is filled in by the CrossDSOCFI pass which + // runs only in the merged module. + return GV->getName() == "__cfi_check"; + }; + ValueToValueMapTy VMap; std::unique_ptr<Module> MergedM( CloneModule(M, VMap, [&](const GlobalValue *GV) -> bool { if (const auto *C = GV->getComdat()) if (MergedMComdats.count(C)) return true; + if (MustEmitToMergedModule(GV)) + return true; if (auto *F = dyn_cast<Function>(GV)) return EligibleVirtualFns.count(F); if (auto *GVar = @@ -372,7 +380,7 @@ void splitAndWriteThinLTOBitcode( cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ true); for (Function &F : *MergedM) - if (!F.isDeclaration()) { + if (!F.isDeclaration() && !MustEmitToMergedModule(&F)) { // Reset the linkage of all functions eligible for virtual constant // propagation. The canonical definitions live in the thin LTO module so // that they can be imported. @@ -394,6 +402,8 @@ void splitAndWriteThinLTOBitcode( if (const auto *C = GV->getComdat()) if (MergedMComdats.count(C)) return false; + if (MustEmitToMergedModule(GV)) + return false; return true; }); diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp index 5b8ea1547ca2..b74a0708b67a 100644 --- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp +++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp @@ -1084,8 +1084,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, auto ThenTerm = SplitBlockAndInsertIfThen( IRB.CreateIsNull(Load), &*IP, false, MDBuilder(IRB.getContext()).createUnlikelyBranchWeights()); - IRBuilder<> ThenIRB(ThenTerm); + InstrumentationIRBuilder ThenIRB(ThenTerm); auto Store = ThenIRB.CreateStore(ConstantInt::getTrue(Int1Ty), FlagPtr); + if (EntryLoc) + Store->setDebugLoc(EntryLoc); Load->setNoSanitizeMetadata(); Store->setNoSanitizeMetadata(); } @@ -1131,7 +1133,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, EstimatedStackSize >= Options.StackDepthCallbackMin) { if (InsertBefore) IRB.SetInsertPoint(InsertBefore); - IRB.CreateCall(SanCovStackDepthCallback)->setCannotMerge(); + auto Call = IRB.CreateCall(SanCovStackDepthCallback); + if (EntryLoc) + Call->setDebugLoc(EntryLoc); + Call->setCannotMerge(); } } else { // Check stack depth. If it's the deepest so far, record it. @@ -1144,8 +1149,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, auto ThenTerm = SplitBlockAndInsertIfThen( IsStackLower, &*IP, false, MDBuilder(IRB.getContext()).createUnlikelyBranchWeights()); - IRBuilder<> ThenIRB(ThenTerm); + InstrumentationIRBuilder ThenIRB(ThenTerm); auto Store = ThenIRB.CreateStore(FrameAddrInt, SanCovLowestStack); + if (EntryLoc) + Store->setDebugLoc(EntryLoc); LowestStack->setNoSanitizeMetadata(); Store->setNoSanitizeMetadata(); } diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp index 85dd9a1bf716..0f63ed0166cf 100644 --- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp +++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp @@ -2079,6 +2079,7 @@ struct DSEState { AllocFnKind AllocKind = Attrs.getFnAttr(Attribute::AllocKind).getAllocKind() | AllocFnKind::Zeroed; + AllocKind &= ~AllocFnKind::Uninitialized; Attrs = Attrs.addFnAttribute(Ctx, Attribute::getWithAllocKind(Ctx, AllocKind)) .removeFnAttribute(Ctx, "alloc-variant-zeroed"); diff --git a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp index 221094f170ac..b9546c5fa236 100644 --- a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp +++ b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp @@ -128,6 +128,8 @@ private: // from any other block. So this variable set to true means that loop's latch // has become unreachable from loop header. bool DeleteCurrentLoop = false; + // Whether or not we enter the loop through an indirectbr. + bool HasIndirectEntry = false; // The blocks of the original loop that will still be reachable from entry // after the constant folding. @@ -216,6 +218,19 @@ private: return; } + // We need a loop preheader to split in handleDeadExits(). If LoopSimplify + // wasn't able to form one because the loop can be entered through an + // indirectbr we cannot continue. + if (!L.getLoopPreheader()) { + assert(any_of(predecessors(L.getHeader()), + [&](BasicBlock *Pred) { + return isa<IndirectBrInst>(Pred->getTerminator()); + }) && + "Loop should have preheader if it is not entered indirectly"); + HasIndirectEntry = true; + return; + } + // Collect live and dead loop blocks and exits. LiveLoopBlocks.insert(L.getHeader()); for (auto I = DFS.beginRPO(), E = DFS.endRPO(); I != E; ++I) { @@ -546,6 +561,12 @@ public: return false; } + if (HasIndirectEntry) { + LLVM_DEBUG(dbgs() << "Loops which can be entered indirectly are not" + " supported!\n"); + return false; + } + // Nothing to constant-fold. if (FoldCandidates.empty()) { LLVM_DEBUG( diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 70b4552190a4..f537b0d3fbd7 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -1258,8 +1258,7 @@ private: "Map index doesn't point back to a slice with this user."); } - // Disable SRoA for any intrinsics except for lifetime invariants and - // invariant group. + // Disable SRoA for any intrinsics except for lifetime invariants. // FIXME: What about debug intrinsics? This matches old behavior, but // doesn't make sense. void visitIntrinsicInst(IntrinsicInst &II) { @@ -1279,12 +1278,6 @@ private: return; } - if (II.isLaunderOrStripInvariantGroup()) { - insertUse(II, Offset, AllocSize, true); - enqueueUsers(II); - return; - } - Base::visitIntrinsicInst(II); } @@ -3618,8 +3611,7 @@ private: } bool visitIntrinsicInst(IntrinsicInst &II) { - assert((II.isLifetimeStartOrEnd() || II.isLaunderOrStripInvariantGroup() || - II.isDroppable()) && + assert((II.isLifetimeStartOrEnd() || II.isDroppable()) && "Unexpected intrinsic!"); LLVM_DEBUG(dbgs() << " original: " << II << "\n"); @@ -3633,9 +3625,6 @@ private: return true; } - if (II.isLaunderOrStripInvariantGroup()) - return true; - assert(II.getArgOperand(1) == OldPtr); // Lifetime intrinsics are only promotable if they cover the whole alloca. // Therefore, we drop lifetime intrinsics which don't cover the whole diff --git a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp index ddd203f3acf7..42b1fdf17f38 100644 --- a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp +++ b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp @@ -111,15 +111,14 @@ BasicBlock * llvm::SplitKnownCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options, const Twine &BBName) { - assert(!isa<IndirectBrInst>(TI) && - "Cannot split critical edge from IndirectBrInst"); - BasicBlock *TIBB = TI->getParent(); BasicBlock *DestBB = TI->getSuccessor(SuccNum); - // Splitting the critical edge to a pad block is non-trivial. Don't do - // it in this generic function. - if (DestBB->isEHPad()) return nullptr; + // Splitting the critical edge to a pad block is non-trivial. + // And we cannot split block with IndirectBr as a terminator. + // Don't do it in this generic function. + if (DestBB->isEHPad() || isa<IndirectBrInst>(TI)) + return nullptr; if (Options.IgnoreUnreachableDests && isa<UnreachableInst>(DestBB->getFirstNonPHIOrDbgOrLifetime())) diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 200d1fb85415..e7623aaff105 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -938,8 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) { case RecurKind::UMin: return Intrinsic::vector_reduce_umin; case RecurKind::FMax: + case RecurKind::FMaxNum: return Intrinsic::vector_reduce_fmax; case RecurKind::FMin: + case RecurKind::FMinNum: return Intrinsic::vector_reduce_fmin; case RecurKind::FMaximum: return Intrinsic::vector_reduce_fmaximum; @@ -1037,8 +1039,10 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) { case RecurKind::SMax: return Intrinsic::smax; case RecurKind::FMin: + case RecurKind::FMinNum: return Intrinsic::minnum; case RecurKind::FMax: + case RecurKind::FMaxNum: return Intrinsic::maxnum; case RecurKind::FMinimum: return Intrinsic::minimum; @@ -1096,9 +1100,9 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right) { Type *Ty = Left->getType(); if (Ty->isIntOrIntVectorTy() || - (RK == RecurKind::FMinimum || RK == RecurKind::FMaximum || + (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum || + RK == RecurKind::FMinimum || RK == RecurKind::FMaximum || RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) { - // TODO: Add float minnum/maxnum support when FMF nnan is set. Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK); return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr, "rdx.minmax"); @@ -1308,6 +1312,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src, case RecurKind::UMin: case RecurKind::FMax: case RecurKind::FMin: + case RecurKind::FMinNum: + case RecurKind::FMaxNum: case RecurKind::FMinimum: case RecurKind::FMaximum: case RecurKind::FMinimumNum: diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 24fe08d6c3e4..bf457194bfd8 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -2133,8 +2133,15 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR, // negative. If Step is known to be positive or negative, only create // either 1. or 2. auto ComputeEndCheck = [&]() -> Value * { - // Checking <u 0 is always false. - if (!Signed && Start->isZero() && SE.isKnownPositive(Step)) + // Checking <u 0 is always false, if (Step * trunc ExitCount) does not wrap. + // TODO: Predicates that can be proven true/false should be discarded when + // the predicates are created, not late during expansion. + if (!Signed && Start->isZero() && SE.isKnownPositive(Step) && + DstBits < SrcBits && + ExitCount == SE.getZeroExtendExpr(SE.getTruncateExpr(ExitCount, ARTy), + ExitCount->getType()) && + SE.willNotOverflow(Instruction::Mul, Signed, Step, + SE.getTruncateExpr(ExitCount, ARTy))) return ConstantInt::getFalse(Loc->getContext()); // Get the backedge taken count and truncate or extended to the AR type. diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 11853859484e..f57ce0c3ccb4 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -230,7 +230,6 @@ public: /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A /// and \p B. - /// TODO: add createFCmp when needed. VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { @@ -240,6 +239,17 @@ public: new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name)); } + /// Create a new FCmp VPInstruction with predicate \p Pred and operands \p A + /// and \p B. + VPInstruction *createFCmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, + DebugLoc DL = DebugLoc::getUnknown(), + const Twine &Name = "") { + assert(Pred >= CmpInst::FIRST_FCMP_PREDICATE && + Pred <= CmpInst::LAST_FCMP_PREDICATE && "invalid predicate"); + return tryInsertInstruction( + new VPInstruction(Instruction::FCmp, {A, B}, Pred, DL, Name)); + } + VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 06db89a89bc3..5cf4b1651538 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4345,10 +4345,14 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization( ElementCount VF) const { - // Cross iteration phis such as reductions need special handling and are - // currently unsupported. - if (any_of(OrigLoop->getHeader()->phis(), - [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); })) + // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum + // reductions need special handling and are currently unsupported. + if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) { + if (!Legal->isReductionVariable(&Phi)) + return Legal->isFixedOrderRecurrence(&Phi); + RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind(); + return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum; + })) return false; // Phis with uses outside of the loop require special handling and are @@ -8817,6 +8821,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( // Adjust the recipes for any inloop reductions. adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start); + // Apply mandatory transformation to handle FP maxnum/minnum reduction with + // NaNs if possible, bail out otherwise. + if (!VPlanTransforms::runPass(VPlanTransforms::handleMaxMinNumReductions, + *Plan)) + return nullptr; + // Transform recipes to abstract recipes if it is legal and beneficial and // clamp the range for better cost estimation. // TODO: Enable following transform when the EVL-version of extended-reduction diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 31aec77db63c..2d50981690f4 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9968,7 +9968,10 @@ BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth, } SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars); if (all_of(VL, [&](Value *V) { - return isa<PoisonValue>(V) || Values.contains(V); + return isa<PoisonValue>(V) || Values.contains(V) || + (S.getOpcode() == Instruction::PHI && isa<PHINode>(V) && + LI->getLoopFor(S.getMainOp()->getParent()) && + isVectorized(V)); })) { LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n"); return ScalarsVectorizationLegality(S, /*IsLegal=*/false); @@ -23196,6 +23199,8 @@ private: case RecurKind::FindFirstIVUMin: case RecurKind::FindLastIVSMax: case RecurKind::FindLastIVUMax: + case RecurKind::FMaxNum: + case RecurKind::FMinNum: case RecurKind::FMaximumNum: case RecurKind::FMinimumNum: case RecurKind::None: @@ -23333,6 +23338,8 @@ private: case RecurKind::FindFirstIVUMin: case RecurKind::FindLastIVSMax: case RecurKind::FindLastIVUMax: + case RecurKind::FMaxNum: + case RecurKind::FMinNum: case RecurKind::FMaximumNum: case RecurKind::FMinimumNum: case RecurKind::None: @@ -23435,6 +23442,8 @@ private: case RecurKind::FindFirstIVUMin: case RecurKind::FindLastIVSMax: case RecurKind::FindLastIVUMax: + case RecurKind::FMaxNum: + case RecurKind::FMinNum: case RecurKind::FMaximumNum: case RecurKind::FMinimumNum: case RecurKind::None: diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index b27a7ffeed20..66657b98b094 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -84,6 +84,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) { return ResTy; } case Instruction::ICmp: + case Instruction::FCmp: case VPInstruction::ActiveLaneMask: assert(inferScalarType(R->getOperand(0)) == inferScalarType(R->getOperand(1)) && diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 52eecb000d0c..a7a22e042aef 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -628,3 +628,163 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond, Term->addMetadata(LLVMContext::MD_prof, BranchWeights); } } + +bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) { + auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * { + auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>( + RedPhiR->getBackedgeValue()->getDefiningRecipe()); + if (!MinMaxR) + return nullptr; + + auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR); + if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) && + !(RepR && isa<IntrinsicInst>(RepR->getUnderlyingInstr()))) + return nullptr; + +#ifndef NDEBUG + Intrinsic::ID RdxIntrinsicId = + RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum + : Intrinsic::minnum; + assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) && + cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() == + RdxIntrinsicId) || + (RepR && + cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() == + RdxIntrinsicId) && + "Intrinsic did not match recurrence kind"); +#endif + + if (MinMaxR->getOperand(0) == RedPhiR) + return MinMaxR->getOperand(1); + + assert(MinMaxR->getOperand(1) == RedPhiR && + "Reduction phi operand expected"); + return MinMaxR->getOperand(0); + }; + + VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); + VPReductionPHIRecipe *RedPhiR = nullptr; + bool HasUnsupportedPhi = false; + for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) { + if (isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe>(&R)) + continue; + auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R); + if (!Cur) { + // TODO: Also support fixed-order recurrence phis. + HasUnsupportedPhi = true; + continue; + } + // For now, only a single reduction is supported. + // TODO: Support multiple MaxNum/MinNum reductions and other reductions. + if (RedPhiR) + return false; + if (Cur->getRecurrenceKind() != RecurKind::FMaxNum && + Cur->getRecurrenceKind() != RecurKind::FMinNum) { + HasUnsupportedPhi = true; + continue; + } + RedPhiR = Cur; + } + + if (!RedPhiR) + return true; + + // We won't be able to resume execution in the scalar tail, if there are + // unsupported header phis or there is no scalar tail at all, due to + // tail-folding. + if (HasUnsupportedPhi || !Plan.hasScalarTail()) + return false; + + VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR); + if (!MinMaxOp) + return false; + + RecurKind RedPhiRK = RedPhiR->getRecurrenceKind(); + assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) && + "unsupported reduction"); + + /// Check if the vector loop of \p Plan can early exit and restart + /// execution of last vector iteration in the scalar loop. This requires all + /// recipes up to early exit point be side-effect free as they are + /// re-executed. Currently we check that the loop is free of any recipe that + /// may write to memory. Expected to operate on an early VPlan w/o nested + /// regions. + for (VPBlockBase *VPB : vp_depth_first_shallow( + Plan.getVectorLoopRegion()->getEntryBasicBlock())) { + auto *VPBB = cast<VPBasicBlock>(VPB); + for (auto &R : *VPBB) { + if (R.mayWriteToMemory() && + !match(&R, m_BranchOnCount(m_VPValue(), m_VPValue()))) + return false; + } + } + + VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock(); + VPBuilder Builder(LatchVPBB->getTerminator()); + auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator()); + assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount && + "Unexpected terminator"); + auto *IsLatchExitTaken = + Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0), + LatchExitingBranch->getOperand(1)); + + VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp); + VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN}); + auto *AnyExitTaken = + Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken}); + Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken); + LatchExitingBranch->eraseFromParent(); + + // If we exit early due to NaNs, compute the final reduction result based on + // the reduction phi at the beginning of the last vector iteration. + auto *RdxResult = find_singleton<VPSingleDefRecipe>( + RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * { + auto *VPI = dyn_cast<VPInstruction>(U); + if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult) + return VPI; + return nullptr; + }); + + auto *MiddleVPBB = Plan.getMiddleBlock(); + Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin()); + auto *NewSel = + Builder.createSelect(AnyNaN, RedPhiR, RdxResult->getOperand(1)); + RdxResult->setOperand(1, NewSel); + + auto *ScalarPH = Plan.getScalarPreheader(); + // Update resume phis for inductions in the scalar preheader. If AnyNaN is + // true, the resume from the start of the last vector iteration via the + // canonical IV, otherwise from the original value. + for (auto &R : ScalarPH->phis()) { + auto *ResumeR = cast<VPPhi>(&R); + VPValue *VecV = ResumeR->getOperand(0); + if (VecV == RdxResult) + continue; + if (auto *DerivedIV = dyn_cast<VPDerivedIVRecipe>(VecV)) { + if (DerivedIV->getNumUsers() == 1 && + DerivedIV->getOperand(1) == &Plan.getVectorTripCount()) { + auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), + &Plan.getVectorTripCount()); + DerivedIV->moveAfter(&*Builder.getInsertPoint()); + DerivedIV->setOperand(1, NewSel); + continue; + } + } + // Bail out and abandon the current, partially modified, VPlan if we + // encounter resume phi that cannot be updated yet. + if (VecV != &Plan.getVectorTripCount()) { + LLVM_DEBUG(dbgs() << "Found resume phi we cannot update for VPlan with " + "FMaxNum/FMinNum reduction.\n"); + return false; + } + auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV); + ResumeR->setOperand(0, NewSel); + } + + auto *MiddleTerm = MiddleVPBB->getTerminator(); + Builder.setInsertPoint(MiddleTerm); + VPValue *MiddleCond = MiddleTerm->getOperand(0); + VPValue *NewCond = Builder.createAnd(MiddleCond, Builder.createNot(AnyNaN)); + MiddleTerm->setOperand(0, NewCond); + return true; +} diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1664bcc3881a..57b713d3dfcb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -587,6 +587,7 @@ Value *VPInstruction::generate(VPTransformState &State) { Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this)); return Builder.CreateFreeze(Op, Name); } + case Instruction::FCmp: case Instruction::ICmp: { bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this); Value *A = State.get(getOperand(0), OnlyFirstLaneUsed); @@ -860,7 +861,7 @@ Value *VPInstruction::generate(VPTransformState &State) { Value *Res = State.get(getOperand(0)); for (VPValue *Op : drop_begin(operands())) Res = Builder.CreateOr(Res, State.get(Op)); - return Builder.CreateOrReduce(Res); + return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res); } case VPInstruction::FirstActiveLane: { if (getNumOperands() == 1) { @@ -1033,6 +1034,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const { switch (getOpcode()) { case Instruction::ExtractElement: case Instruction::Freeze: + case Instruction::FCmp: case Instruction::ICmp: case Instruction::Select: case VPInstruction::AnyOf: @@ -1068,6 +1070,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const { return Op == getOperand(1); case Instruction::PHI: return true; + case Instruction::FCmp: case Instruction::ICmp: case Instruction::Select: case Instruction::Or: @@ -1100,6 +1103,7 @@ bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const { switch (getOpcode()) { default: return false; + case Instruction::FCmp: case Instruction::ICmp: case Instruction::Select: return vputils::onlyFirstPartUsed(this); @@ -1786,7 +1790,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const { return Opcode == Instruction::ZExt; break; case OperationType::Cmp: - return Opcode == Instruction::ICmp; + return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp; case OperationType::Other: return true; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6a3b3e6e4195..f7c1c10185c6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3252,9 +3252,10 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, return; // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe. - auto NarrowOp = [](VPValue *V) -> VPValue * { + SmallPtrSet<VPValue *, 4> NarrowedOps; + auto NarrowOp = [&NarrowedOps](VPValue *V) -> VPValue * { auto *R = V->getDefiningRecipe(); - if (!R) + if (!R || NarrowedOps.contains(V)) return V; if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(R)) { // Narrow interleave group to wide load, as transformed VPlan will only @@ -3264,6 +3265,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true, /*Reverse=*/false, {}, LoadGroup->getDebugLoc()); L->insertBefore(LoadGroup); + NarrowedOps.insert(L); return L; } @@ -3271,6 +3273,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, assert(RepR->isSingleScalar() && isa<LoadInst>(RepR->getUnderlyingInstr()) && "must be a single scalar load"); + NarrowedOps.insert(RepR); return RepR; } auto *WideLoad = cast<VPWidenLoadRecipe>(R); @@ -3281,6 +3284,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, WideLoad->operands(), /*IsUniform*/ true, /*Mask*/ nullptr, *WideLoad); N->insertBefore(WideLoad); + NarrowedOps.insert(N); return N; }; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 870b1bb68b79..c3fb359d1429 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -99,6 +99,12 @@ struct VPlanTransforms { /// not valid. static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder); + /// Check if \p Plan contains any FMaxNum or FMinNum reductions. If they do, + /// try to update the vector loop to exit early if any input is NaN and resume + /// executing in the scalar loop to handle the NaNs there. Return false if + /// this attempt was unsuccessful. + static bool handleMaxMinNumReductions(VPlan &Plan); + /// Clear NSW/NUW flags from reduction instructions if necessary. static void clearReductionWrapFlags(VPlan &Plan); diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index fe8d74c43dfd..ea9cbed0117b 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -74,7 +74,7 @@ public: const DataLayout *DL, TTI::TargetCostKind CostKind, bool TryEarlyFoldsOnly) : F(F), Builder(F.getContext(), InstSimplifyFolder(*DL)), TTI(TTI), - DT(DT), AA(AA), AC(AC), DL(DL), CostKind(CostKind), + DT(DT), AA(AA), AC(AC), DL(DL), CostKind(CostKind), SQ(*DL), TryEarlyFoldsOnly(TryEarlyFoldsOnly) {} bool run(); @@ -88,6 +88,7 @@ private: AssumptionCache &AC; const DataLayout *DL; TTI::TargetCostKind CostKind; + const SimplifyQuery SQ; /// If true, only perform beneficial early IR transforms. Do not introduce new /// vector operations. @@ -1185,17 +1186,18 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) { // Fold the vector constants in the original vectors into a new base vector to // get more accurate cost modelling. Value *NewVecC = nullptr; - TargetFolder Folder(*DL); if (CI) - NewVecC = Folder.FoldCmp(CI->getPredicate(), VecCs[0], VecCs[1]); + NewVecC = simplifyCmpInst(CI->getPredicate(), VecCs[0], VecCs[1], SQ); else if (UO) NewVecC = - Folder.FoldUnOpFMF(UO->getOpcode(), VecCs[0], UO->getFastMathFlags()); + simplifyUnOp(UO->getOpcode(), VecCs[0], UO->getFastMathFlags(), SQ); else if (BO) - NewVecC = Folder.FoldBinOp(BO->getOpcode(), VecCs[0], VecCs[1]); - else if (II->arg_size() == 2) - NewVecC = Folder.FoldBinaryIntrinsic(II->getIntrinsicID(), VecCs[0], - VecCs[1], II->getType(), &I); + NewVecC = simplifyBinOp(BO->getOpcode(), VecCs[0], VecCs[1], SQ); + else if (II) + NewVecC = simplifyCall(II, II->getCalledOperand(), VecCs, SQ); + + if (!NewVecC) + return false; // Get cost estimate for the insert element. This cost will factor into // both sequences. @@ -1203,6 +1205,7 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) { InstructionCost NewCost = ScalarOpCost + TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, *Index, NewVecC); + for (auto [Idx, Op, VecC, Scalar] : enumerate(Ops, VecCs, ScalarOps)) { if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg( II->getIntrinsicID(), Idx, &TTI))) @@ -1247,15 +1250,6 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) { if (auto *ScalarInst = dyn_cast<Instruction>(Scalar)) ScalarInst->copyIRFlags(&I); - // Create a new base vector if the constant folding failed. - if (!NewVecC) { - if (CI) - NewVecC = Builder.CreateCmp(CI->getPredicate(), VecCs[0], VecCs[1]); - else if (UO || BO) - NewVecC = Builder.CreateNAryOp(Opcode, VecCs); - else - NewVecC = Builder.CreateIntrinsic(VecTy, II->getIntrinsicID(), VecCs); - } Value *Insert = Builder.CreateInsertElement(NewVecC, Scalar, *Index); replaceValue(I, *Insert); return true; @@ -1835,12 +1829,19 @@ bool VectorCombine::scalarizeExtExtract(Instruction &I) { IntegerType::get(SrcTy->getContext(), DL->getTypeSizeInBits(SrcTy))); uint64_t SrcEltSizeInBits = DL->getTypeSizeInBits(SrcTy->getElementType()); uint64_t EltBitMask = (1ull << SrcEltSizeInBits) - 1; + uint64_t TotalBits = DL->getTypeSizeInBits(SrcTy); + Type *PackedTy = IntegerType::get(SrcTy->getContext(), TotalBits); + Value *Mask = ConstantInt::get(PackedTy, EltBitMask); for (User *U : Ext->users()) { auto *Extract = cast<ExtractElementInst>(U); uint64_t Idx = cast<ConstantInt>(Extract->getIndexOperand())->getZExtValue(); - Value *LShr = Builder.CreateLShr(ScalarV, Idx * SrcEltSizeInBits); - Value *And = Builder.CreateAnd(LShr, EltBitMask); + uint64_t ShiftAmt = + DL->isBigEndian() + ? (TotalBits - SrcEltSizeInBits - Idx * SrcEltSizeInBits) + : (Idx * SrcEltSizeInBits); + Value *LShr = Builder.CreateLShr(ScalarV, ShiftAmt); + Value *And = Builder.CreateAnd(LShr, Mask); U->replaceAllUsesWith(And); } return true; diff --git a/llvm/tools/llvm-mc/Disassembler.cpp b/llvm/tools/llvm-mc/Disassembler.cpp index 607184e3b724..86727931067a 100644 --- a/llvm/tools/llvm-mc/Disassembler.cpp +++ b/llvm/tools/llvm-mc/Disassembler.cpp @@ -45,7 +45,11 @@ static bool PrintInsts(const MCDisassembler &DisAsm, const ByteArrayTy &Bytes, MCInst Inst; MCDisassembler::DecodeStatus S; - S = DisAsm.getInstruction(Inst, Size, Data.slice(Index), Index, nulls()); + if (STI.getTargetTriple().getArch() == Triple::hexagon) + S = DisAsm.getInstructionBundle(Inst, Size, Data.slice(Index), Index, + nulls()); + else + S = DisAsm.getInstruction(Inst, Size, Data.slice(Index), Index, nulls()); switch (S) { case MCDisassembler::Fail: SM.PrintMessage(SMLoc::getFromPointer(Bytes.second[Index]), diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp index c5967cd090ee..221b884e0c06 100644 --- a/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -693,11 +693,41 @@ public: } else OS << "\t<unknown>"; } + + virtual void emitPostInstructionInfo(formatted_raw_ostream &FOS, + const MCAsmInfo &MAI, + const MCSubtargetInfo &STI, + StringRef Comments, + LiveVariablePrinter &LVP) { + do { + if (!Comments.empty()) { + // Emit a line of comments. + StringRef Comment; + std::tie(Comment, Comments) = Comments.split('\n'); + // MAI.getCommentColumn() assumes that instructions are printed at the + // position of 8, while getInstStartColumn() returns the actual + // position. + unsigned CommentColumn = + MAI.getCommentColumn() - 8 + getInstStartColumn(STI); + FOS.PadToColumn(CommentColumn); + FOS << MAI.getCommentString() << ' ' << Comment; + } + LVP.printAfterInst(FOS); + FOS << "\n"; + } while (!Comments.empty()); + FOS.flush(); + } + + // Hook invoked when starting to disassemble a symbol at the current position. + // Default is no-op. + virtual void onSymbolStart() {} }; PrettyPrinter PrettyPrinterInst; class HexagonPrettyPrinter : public PrettyPrinter { public: + void onSymbolStart() override { reset(); } + void printLead(ArrayRef<uint8_t> Bytes, uint64_t Address, formatted_raw_ostream &OS) { if (LeadingAddr) @@ -714,6 +744,35 @@ public: } } } + + std::string getInstructionSeparator() const { + SmallString<40> Separator; + raw_svector_ostream OS(Separator); + if (ShouldClosePacket) { + OS << " }"; + if (IsLoop0 || IsLoop1) + OS << " "; + if (IsLoop0) + OS << (IsLoop1 ? ":endloop01" : ":endloop0"); + else if (IsLoop1) + OS << ":endloop1"; + } + OS << '\n'; + return OS.str().str(); + } + + void emitPostInstructionInfo(formatted_raw_ostream &FOS, const MCAsmInfo &MAI, + const MCSubtargetInfo &STI, StringRef Comments, + LiveVariablePrinter &LVP) override { + // Hexagon does not write anything to the comment stream, so we can just + // print the separator. + LVP.printAfterInst(FOS); + FOS << getInstructionSeparator(); + FOS.flush(); + if (ShouldClosePacket) + reset(); + } + void printInst(MCInstPrinter &IP, const MCInst *MI, ArrayRef<uint8_t> Bytes, object::SectionedAddress Address, formatted_raw_ostream &OS, StringRef Annot, MCSubtargetInfo const &STI, SourcePrinter *SP, @@ -724,60 +783,64 @@ public: if (!MI) { printLead(Bytes, Address.Address, OS); OS << " <unknown>"; + reset(); return; } - std::string Buffer; + + StringRef Preamble = IsStartOfBundle ? " { " : " "; + + if (SP && (PrintSource || PrintLines)) + SP->printSourceLine(OS, Address, ObjectFilename, LVP, ""); + printLead(Bytes, Address.Address, OS); + OS << Preamble; + std::string Buf; { - raw_string_ostream TempStream(Buffer); + raw_string_ostream TempStream(Buf); IP.printInst(MI, Address.Address, "", STI, TempStream); } - StringRef Contents(Buffer); - // Split off bundle attributes - auto PacketBundle = Contents.rsplit('\n'); - // Split off first instruction from the rest - auto HeadTail = PacketBundle.first.split('\n'); - auto Preamble = " { "; - auto Separator = ""; - - // Hexagon's packets require relocations to be inline rather than - // clustered at the end of the packet. - std::vector<RelocationRef>::const_iterator RelCur = Rels->begin(); - std::vector<RelocationRef>::const_iterator RelEnd = Rels->end(); - auto PrintReloc = [&]() -> void { - while ((RelCur != RelEnd) && (RelCur->getOffset() <= Address.Address)) { - if (RelCur->getOffset() == Address.Address) { - printRelocation(OS, ObjectFilename, *RelCur, Address.Address, false); - return; - } - ++RelCur; - } - }; + StringRef Contents(Buf); + + auto Duplex = Contents.split('\v'); + bool HasDuplex = !Duplex.second.empty(); + if (HasDuplex) { + OS << Duplex.first; + OS << "; "; + OS << Duplex.second; + } else { + OS << Duplex.first; + } - while (!HeadTail.first.empty()) { - OS << Separator; - Separator = "\n"; - if (SP && (PrintSource || PrintLines)) - SP->printSourceLine(OS, Address, ObjectFilename, LVP, ""); - printLead(Bytes, Address.Address, OS); - OS << Preamble; - Preamble = " "; - StringRef Inst; - auto Duplex = HeadTail.first.split('\v'); - if (!Duplex.second.empty()) { - OS << Duplex.first; - OS << "; "; - Inst = Duplex.second; - } + uint32_t Instruction = support::endian::read32le(Bytes.data()); + + uint32_t ParseMask = 0x0000c000; + uint32_t PacketEndMask = 0x0000c000; + uint32_t LoopEndMask = 0x00008000; + uint32_t ParseBits = Instruction & ParseMask; + + if (ParseBits == LoopEndMask) { + if (IsStartOfBundle) + IsLoop0 = true; else - Inst = HeadTail.first; - OS << Inst; - HeadTail = HeadTail.second.split('\n'); - if (HeadTail.first.empty()) - OS << " } " << PacketBundle.second; - PrintReloc(); - Bytes = Bytes.slice(4); - Address.Address += 4; + IsLoop1 = true; } + + IsStartOfBundle = false; + + if (ParseBits == PacketEndMask || HasDuplex) + ShouldClosePacket = true; + } + +private: + bool IsStartOfBundle = true; + bool IsLoop0 = false; + bool IsLoop1 = false; + bool ShouldClosePacket = false; + + void reset() { + IsStartOfBundle = true; + IsLoop0 = false; + IsLoop1 = false; + ShouldClosePacket = false; } }; HexagonPrettyPrinter HexagonPrettyPrinterInst; @@ -1610,29 +1673,6 @@ static StringRef getSegmentName(const MachOObjectFile *MachO, return ""; } -static void emitPostInstructionInfo(formatted_raw_ostream &FOS, - const MCAsmInfo &MAI, - const MCSubtargetInfo &STI, - StringRef Comments, - LiveVariablePrinter &LVP) { - do { - if (!Comments.empty()) { - // Emit a line of comments. - StringRef Comment; - std::tie(Comment, Comments) = Comments.split('\n'); - // MAI.getCommentColumn() assumes that instructions are printed at the - // position of 8, while getInstStartColumn() returns the actual position. - unsigned CommentColumn = - MAI.getCommentColumn() - 8 + getInstStartColumn(STI); - FOS.PadToColumn(CommentColumn); - FOS << MAI.getCommentString() << ' ' << Comment; - } - LVP.printAfterInst(FOS); - FOS << '\n'; - } while (!Comments.empty()); - FOS.flush(); -} - static void createFakeELFSections(ObjectFile &Obj) { assert(Obj.isELF()); if (auto *Elf32LEObj = dyn_cast<ELF32LEObjectFile>(&Obj)) @@ -2182,6 +2222,8 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, Start += Size; break; } + // Allow targets to reset any per-symbol state. + DT->Printer->onSymbolStart(); formatted_raw_ostream FOS(OS); Index = Start; if (SectionAddr < StartAddress) @@ -2526,15 +2568,15 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, } assert(DT->Context->getAsmInfo()); - emitPostInstructionInfo(FOS, *DT->Context->getAsmInfo(), - *DT->SubtargetInfo, CommentStream.str(), LVP); + DT->Printer->emitPostInstructionInfo(FOS, *DT->Context->getAsmInfo(), + *DT->SubtargetInfo, + CommentStream.str(), LVP); Comments.clear(); if (BTF) printBTFRelocation(FOS, *BTF, {Index, Section.getIndex()}, LVP); - // Hexagon handles relocs in pretty printer - if (InlineRelocs && Obj.getArch() != Triple::hexagon) { + if (InlineRelocs) { while (findRel()) { // When --adjust-vma is used, update the address printed. printRelocation(FOS, Obj.getFileName(), *RelCur, diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index 101079f09e1d..3fd167df1ecc 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -1083,26 +1083,26 @@ const EnumEntry<unsigned> ElfObjectFileType[] = { }; const EnumEntry<unsigned> ElfOSABI[] = { - {"SystemV", "UNIX - System V", ELF::ELFOSABI_NONE}, - {"HPUX", "UNIX - HP-UX", ELF::ELFOSABI_HPUX}, - {"NetBSD", "UNIX - NetBSD", ELF::ELFOSABI_NETBSD}, - {"GNU/Linux", "UNIX - GNU", ELF::ELFOSABI_LINUX}, - {"GNU/Hurd", "GNU/Hurd", ELF::ELFOSABI_HURD}, - {"Solaris", "UNIX - Solaris", ELF::ELFOSABI_SOLARIS}, - {"AIX", "UNIX - AIX", ELF::ELFOSABI_AIX}, - {"IRIX", "UNIX - IRIX", ELF::ELFOSABI_IRIX}, - {"FreeBSD", "UNIX - FreeBSD", ELF::ELFOSABI_FREEBSD}, - {"TRU64", "UNIX - TRU64", ELF::ELFOSABI_TRU64}, - {"Modesto", "Novell - Modesto", ELF::ELFOSABI_MODESTO}, - {"OpenBSD", "UNIX - OpenBSD", ELF::ELFOSABI_OPENBSD}, - {"OpenVMS", "VMS - OpenVMS", ELF::ELFOSABI_OPENVMS}, - {"NSK", "HP - Non-Stop Kernel", ELF::ELFOSABI_NSK}, - {"AROS", "AROS", ELF::ELFOSABI_AROS}, - {"FenixOS", "FenixOS", ELF::ELFOSABI_FENIXOS}, - {"CloudABI", "CloudABI", ELF::ELFOSABI_CLOUDABI}, - {"CUDA", "NVIDIA - CUDA", ELF::ELFOSABI_CUDA}, - {"Standalone", "Standalone App", ELF::ELFOSABI_STANDALONE} -}; + {"SystemV", "UNIX - System V", ELF::ELFOSABI_NONE}, + {"HPUX", "UNIX - HP-UX", ELF::ELFOSABI_HPUX}, + {"NetBSD", "UNIX - NetBSD", ELF::ELFOSABI_NETBSD}, + {"GNU/Linux", "UNIX - GNU", ELF::ELFOSABI_LINUX}, + {"GNU/Hurd", "GNU/Hurd", ELF::ELFOSABI_HURD}, + {"Solaris", "UNIX - Solaris", ELF::ELFOSABI_SOLARIS}, + {"AIX", "UNIX - AIX", ELF::ELFOSABI_AIX}, + {"IRIX", "UNIX - IRIX", ELF::ELFOSABI_IRIX}, + {"FreeBSD", "UNIX - FreeBSD", ELF::ELFOSABI_FREEBSD}, + {"TRU64", "UNIX - TRU64", ELF::ELFOSABI_TRU64}, + {"Modesto", "Novell - Modesto", ELF::ELFOSABI_MODESTO}, + {"OpenBSD", "UNIX - OpenBSD", ELF::ELFOSABI_OPENBSD}, + {"OpenVMS", "VMS - OpenVMS", ELF::ELFOSABI_OPENVMS}, + {"NSK", "HP - Non-Stop Kernel", ELF::ELFOSABI_NSK}, + {"AROS", "AROS", ELF::ELFOSABI_AROS}, + {"FenixOS", "FenixOS", ELF::ELFOSABI_FENIXOS}, + {"CloudABI", "CloudABI", ELF::ELFOSABI_CLOUDABI}, + {"CUDA", "NVIDIA - CUDA", ELF::ELFOSABI_CUDA}, + {"CUDA", "NVIDIA - CUDA", ELF::ELFOSABI_CUDA_V2}, + {"Standalone", "Standalone App", ELF::ELFOSABI_STANDALONE}}; const EnumEntry<unsigned> AMDGPUElfOSABI[] = { {"AMDGPU_HSA", "AMDGPU - HSA", ELF::ELFOSABI_AMDGPU_HSA}, @@ -1666,16 +1666,60 @@ const EnumEntry<unsigned> ElfHeaderAMDGPUFlagsABIVersion4[] = { }; const EnumEntry<unsigned> ElfHeaderNVPTXFlags[] = { - ENUM_ENT(EF_CUDA_SM20, "sm_20"), ENUM_ENT(EF_CUDA_SM21, "sm_21"), - ENUM_ENT(EF_CUDA_SM30, "sm_30"), ENUM_ENT(EF_CUDA_SM32, "sm_32"), - ENUM_ENT(EF_CUDA_SM35, "sm_35"), ENUM_ENT(EF_CUDA_SM37, "sm_37"), - ENUM_ENT(EF_CUDA_SM50, "sm_50"), ENUM_ENT(EF_CUDA_SM52, "sm_52"), - ENUM_ENT(EF_CUDA_SM53, "sm_53"), ENUM_ENT(EF_CUDA_SM60, "sm_60"), - ENUM_ENT(EF_CUDA_SM61, "sm_61"), ENUM_ENT(EF_CUDA_SM62, "sm_62"), - ENUM_ENT(EF_CUDA_SM70, "sm_70"), ENUM_ENT(EF_CUDA_SM72, "sm_72"), - ENUM_ENT(EF_CUDA_SM75, "sm_75"), ENUM_ENT(EF_CUDA_SM80, "sm_80"), - ENUM_ENT(EF_CUDA_SM86, "sm_86"), ENUM_ENT(EF_CUDA_SM87, "sm_87"), - ENUM_ENT(EF_CUDA_SM89, "sm_89"), ENUM_ENT(EF_CUDA_SM90, "sm_90"), + ENUM_ENT(EF_CUDA_SM20, "sm_20"), + ENUM_ENT(EF_CUDA_SM21, "sm_21"), + ENUM_ENT(EF_CUDA_SM30, "sm_30"), + ENUM_ENT(EF_CUDA_SM32, "sm_32"), + ENUM_ENT(EF_CUDA_SM35, "sm_35"), + ENUM_ENT(EF_CUDA_SM37, "sm_37"), + ENUM_ENT(EF_CUDA_SM50, "sm_50"), + ENUM_ENT(EF_CUDA_SM52, "sm_52"), + ENUM_ENT(EF_CUDA_SM53, "sm_53"), + ENUM_ENT(EF_CUDA_SM60, "sm_60"), + ENUM_ENT(EF_CUDA_SM61, "sm_61"), + ENUM_ENT(EF_CUDA_SM62, "sm_62"), + ENUM_ENT(EF_CUDA_SM70, "sm_70"), + ENUM_ENT(EF_CUDA_SM72, "sm_72"), + ENUM_ENT(EF_CUDA_SM75, "sm_75"), + ENUM_ENT(EF_CUDA_SM80, "sm_80"), + ENUM_ENT(EF_CUDA_SM86, "sm_86"), + ENUM_ENT(EF_CUDA_SM87, "sm_87"), + ENUM_ENT(EF_CUDA_SM88, "sm_88"), + ENUM_ENT(EF_CUDA_SM89, "sm_89"), + ENUM_ENT(EF_CUDA_SM90, "sm_90"), + ENUM_ENT(EF_CUDA_SM100, "sm_100"), + ENUM_ENT(EF_CUDA_SM101, "sm_101"), + ENUM_ENT(EF_CUDA_SM103, "sm_103"), + ENUM_ENT(EF_CUDA_SM110, "sm_110"), + ENUM_ENT(EF_CUDA_SM120, "sm_120"), + ENUM_ENT(EF_CUDA_SM121, "sm_121"), + ENUM_ENT(EF_CUDA_SM20 << EF_CUDA_SM_OFFSET, "sm_20"), + ENUM_ENT(EF_CUDA_SM21 << EF_CUDA_SM_OFFSET, "sm_21"), + ENUM_ENT(EF_CUDA_SM30 << EF_CUDA_SM_OFFSET, "sm_30"), + ENUM_ENT(EF_CUDA_SM32 << EF_CUDA_SM_OFFSET, "sm_32"), + ENUM_ENT(EF_CUDA_SM35 << EF_CUDA_SM_OFFSET, "sm_35"), + ENUM_ENT(EF_CUDA_SM37 << EF_CUDA_SM_OFFSET, "sm_37"), + ENUM_ENT(EF_CUDA_SM50 << EF_CUDA_SM_OFFSET, "sm_50"), + ENUM_ENT(EF_CUDA_SM52 << EF_CUDA_SM_OFFSET, "sm_52"), + ENUM_ENT(EF_CUDA_SM53 << EF_CUDA_SM_OFFSET, "sm_53"), + ENUM_ENT(EF_CUDA_SM60 << EF_CUDA_SM_OFFSET, "sm_60"), + ENUM_ENT(EF_CUDA_SM61 << EF_CUDA_SM_OFFSET, "sm_61"), + ENUM_ENT(EF_CUDA_SM62 << EF_CUDA_SM_OFFSET, "sm_62"), + ENUM_ENT(EF_CUDA_SM70 << EF_CUDA_SM_OFFSET, "sm_70"), + ENUM_ENT(EF_CUDA_SM72 << EF_CUDA_SM_OFFSET, "sm_72"), + ENUM_ENT(EF_CUDA_SM75 << EF_CUDA_SM_OFFSET, "sm_75"), + ENUM_ENT(EF_CUDA_SM80 << EF_CUDA_SM_OFFSET, "sm_80"), + ENUM_ENT(EF_CUDA_SM86 << EF_CUDA_SM_OFFSET, "sm_86"), + ENUM_ENT(EF_CUDA_SM87 << EF_CUDA_SM_OFFSET, "sm_87"), + ENUM_ENT(EF_CUDA_SM88 << EF_CUDA_SM_OFFSET, "sm_88"), + ENUM_ENT(EF_CUDA_SM89 << EF_CUDA_SM_OFFSET, "sm_89"), + ENUM_ENT(EF_CUDA_SM90 << EF_CUDA_SM_OFFSET, "sm_90"), + ENUM_ENT(EF_CUDA_SM100 << EF_CUDA_SM_OFFSET, "sm_100"), + ENUM_ENT(EF_CUDA_SM101 << EF_CUDA_SM_OFFSET, "sm_101"), + ENUM_ENT(EF_CUDA_SM103 << EF_CUDA_SM_OFFSET, "sm_103"), + ENUM_ENT(EF_CUDA_SM110 << EF_CUDA_SM_OFFSET, "sm_110"), + ENUM_ENT(EF_CUDA_SM120 << EF_CUDA_SM_OFFSET, "sm_120"), + ENUM_ENT(EF_CUDA_SM121 << EF_CUDA_SM_OFFSET, "sm_121"), }; const EnumEntry<unsigned> ElfHeaderRISCVFlags[] = { @@ -3650,10 +3694,16 @@ template <class ELFT> void GNUELFDumper<ELFT>::printFileHeaders() { else if (e.e_machine == EM_XTENSA) ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderXtensaFlags), unsigned(ELF::EF_XTENSA_MACH)); - else if (e.e_machine == EM_CUDA) + else if (e.e_machine == EM_CUDA) { ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderNVPTXFlags), unsigned(ELF::EF_CUDA_SM)); - else if (e.e_machine == EM_AMDGPU) { + if (e.e_ident[ELF::EI_ABIVERSION] == ELF::ELFABIVERSION_CUDA_V1 && + (e.e_flags & ELF::EF_CUDA_ACCELERATORS_V1)) + ElfFlags += "a"; + else if (e.e_ident[ELF::EI_ABIVERSION] == ELF::ELFABIVERSION_CUDA_V2 && + (e.e_flags & ELF::EF_CUDA_ACCELERATORS)) + ElfFlags += "a"; + } else if (e.e_machine == EM_AMDGPU) { switch (e.e_ident[ELF::EI_ABIVERSION]) { default: break; diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h index f62cabee6ea8..197cf5476528 100644 --- a/openmp/runtime/src/kmp.h +++ b/openmp/runtime/src/kmp.h @@ -106,12 +106,15 @@ class kmp_stats_list; // OMPD_SKIP_HWLOC used in libompd/omp-icv.cpp to avoid OMPD depending on hwloc #if KMP_USE_HWLOC && KMP_AFFINITY_SUPPORTED && !defined(OMPD_SKIP_HWLOC) #include "hwloc.h" +#define KMP_HWLOC_ENABLED 1 #ifndef HWLOC_OBJ_NUMANODE #define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE #endif #ifndef HWLOC_OBJ_PACKAGE #define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET #endif +#else +#define KMP_HWLOC_ENABLED 0 #endif #if KMP_ARCH_X86 || KMP_ARCH_X86_64 @@ -692,10 +695,10 @@ typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *, extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity; #endif /* KMP_OS_WINDOWS */ -#if KMP_USE_HWLOC && !defined(OMPD_SKIP_HWLOC) +#if KMP_HWLOC_ENABLED extern hwloc_topology_t __kmp_hwloc_topology; extern int __kmp_hwloc_error; -#endif +#endif // KMP_HWLOC_ENABLED extern size_t __kmp_affin_mask_size; #define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0) @@ -804,10 +807,10 @@ public: static void destroy_api(); enum api_type { NATIVE_OS -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED , HWLOC -#endif +#endif // KMP_HWLOC_ENABLED }; virtual api_type get_api_type() const { KMP_ASSERT(0); @@ -876,9 +879,9 @@ enum affinity_top_method { affinity_top_method_group, #endif /* KMP_GROUP_AFFINITY */ affinity_top_method_flat, -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED affinity_top_method_hwloc, -#endif +#endif // KMP_HWLOC_ENABLED affinity_top_method_default }; @@ -1145,9 +1148,9 @@ typedef struct kmp_allocator_t { omp_alloctrait_value_t target_access; omp_alloctrait_value_t atomic_scope; size_t part_size; -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED omp_alloctrait_value_t membind; -#endif +#endif // KMP_HWLOC_ENABLED } kmp_allocator_t; extern omp_allocator_handle_t __kmpc_init_allocator(int gtid, @@ -2107,12 +2110,12 @@ typedef struct dispatch_shared_info { #if KMP_USE_HIER_SCHED void *hier; #endif -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED // When linking with libhwloc, the ORDERED EPCC test slows down on big // machines (> 48 cores). Performance analysis showed that a cache thrash // was occurring and this padding helps alleviate the problem. char padding[64]; -#endif +#endif // KMP_HWLOC_ENABLED } dispatch_shared_info_t; typedef struct kmp_disp { diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp index a6065fe792d5..50389502d3b4 100644 --- a/openmp/runtime/src/kmp_affinity.cpp +++ b/openmp/runtime/src/kmp_affinity.cpp @@ -19,13 +19,13 @@ #if KMP_USE_HIER_SCHED #include "kmp_dispatch_hier.h" #endif -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED // Copied from hwloc #define HWLOC_GROUP_KIND_INTEL_MODULE 102 #define HWLOC_GROUP_KIND_INTEL_TILE 103 #define HWLOC_GROUP_KIND_INTEL_DIE 104 #define HWLOC_GROUP_KIND_WINDOWS_PROCESSOR_GROUP 220 -#endif +#endif // KMP_HWLOC_ENABLED #include <ctype.h> // The machine topology @@ -1438,7 +1438,7 @@ void KMPAffinity::pick_api() { KMPAffinity *affinity_dispatch; if (picked_api) return; -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED // Only use Hwloc if affinity isn't explicitly disabled and // user requests Hwloc topology method if (__kmp_affinity_top_method == affinity_top_method_hwloc && @@ -1446,7 +1446,7 @@ void KMPAffinity::pick_api() { affinity_dispatch = new KMPHwlocAffinity(); __kmp_hwloc_available = true; } else -#endif +#endif // KMP_HWLOC_ENABLED { affinity_dispatch = new KMPNativeAffinity(); } @@ -1697,7 +1697,7 @@ kmp_affin_mask_t *__kmp_affin_fullMask = NULL; // Original mask is a subset of full mask in multiple processor groups topology kmp_affin_mask_t *__kmp_affin_origMask = NULL; -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED static inline bool __kmp_hwloc_is_cache_type(hwloc_obj_t obj) { #if HWLOC_API_VERSION >= 0x00020000 return hwloc_obj_type_is_cache(obj->type); @@ -2005,7 +2005,7 @@ static bool __kmp_affinity_create_hwloc_map(kmp_i18n_id_t *const msg_id) { __kmp_topology->sort_ids(); return true; } -#endif // KMP_USE_HWLOC +#endif // KMP_HWLOC_ENABLED // If we don't know how to retrieve the machine's processor topology, or // encounter an error in doing so, this routine is called to form a "flat" @@ -4845,7 +4845,7 @@ static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) { // In the default code path, errors are not fatal - we just try using // another method. We only emit a warning message if affinity is on, or the // verbose flag is set, an the nowarnings flag was not set. -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED if (!success && __kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) { if (!__kmp_hwloc_error) { @@ -4857,7 +4857,7 @@ static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) { KMP_INFORM(AffIgnoringHwloc, env_var); } } -#endif +#endif // KMP_HWLOC_ENABLED #if KMP_ARCH_X86 || KMP_ARCH_X86_64 if (!success) { @@ -4905,7 +4905,7 @@ static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) { // If the user has specified that a paricular topology discovery method is to be // used, then we abort if that method fails. The exception is group affinity, // which might have been implicitly set. -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED else if (__kmp_affinity_top_method == affinity_top_method_hwloc) { KMP_ASSERT(__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC); success = __kmp_affinity_create_hwloc_map(&msg_id); @@ -4914,7 +4914,7 @@ static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) { KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id)); } } -#endif // KMP_USE_HWLOC +#endif // KMP_HWLOC_ENABLED #if KMP_ARCH_X86 || KMP_ARCH_X86_64 else if (__kmp_affinity_top_method == affinity_top_method_x2apicid || @@ -5308,12 +5308,12 @@ void __kmp_affinity_uninitialize(void) { __kmp_free(__kmp_osid_to_hwthread_map); __kmp_osid_to_hwthread_map = NULL; } -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED if (__kmp_hwloc_topology != NULL) { hwloc_topology_destroy(__kmp_hwloc_topology); __kmp_hwloc_topology = NULL; } -#endif +#endif // KMP_HWLOC_ENABLED if (__kmp_hw_subset) { kmp_hw_subset_t::deallocate(__kmp_hw_subset); __kmp_hw_subset = nullptr; diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h index dc3191caae63..fa69585f7e2d 100644 --- a/openmp/runtime/src/kmp_affinity.h +++ b/openmp/runtime/src/kmp_affinity.h @@ -18,7 +18,7 @@ #include <limits> #if KMP_AFFINITY_SUPPORTED -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED class KMPHwlocAffinity : public KMPAffinity { public: class Mask : public KMPAffinity::Mask { @@ -109,7 +109,7 @@ public: } return error; } -#endif +#endif // KMP_OS_WINDOWS int get_proc_group() const override { int group = -1; #if KMP_OS_WINDOWS @@ -191,7 +191,7 @@ public: } api_type get_api_type() const override { return HWLOC; } }; -#endif /* KMP_USE_HWLOC */ +#endif /* KMP_HWLOC_ENABLED */ #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ KMP_OS_AIX diff --git a/openmp/runtime/src/kmp_alloc.cpp b/openmp/runtime/src/kmp_alloc.cpp index 051f88c5a099..d43daefb6ef1 100644 --- a/openmp/runtime/src/kmp_alloc.cpp +++ b/openmp/runtime/src/kmp_alloc.cpp @@ -14,7 +14,7 @@ #include "kmp_io.h" #include "kmp_wrapper_malloc.h" -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED #if HWLOC_API_VERSION > 0x00020300 #define KMP_HWLOC_LOCATION_TYPE_CPUSET HWLOC_LOCATION_TYPE_CPUSET #elif HWLOC_API_VERSION == 0x00020300 @@ -26,7 +26,7 @@ enum hwloc_memattr_id_e { HWLOC_MEMATTR_ID_CAPACITY }; #endif -#endif // KMP_USE_HWLOC +#endif // KMP_HWLOC_ENABLED // Disable bget when it is not used #if KMP_USE_BGET @@ -1545,7 +1545,7 @@ void __kmp_fini_memkind() { #endif } -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED static bool __kmp_is_hwloc_membind_supported(hwloc_membind_policy_t policy) { #if HWLOC_API_VERSION >= 0x00020300 const hwloc_topology_support *support; @@ -1561,7 +1561,7 @@ static bool __kmp_is_hwloc_membind_supported(hwloc_membind_policy_t policy) { return false; #else return false; -#endif +#endif // KMP_HWLOC_ENABLED } void *__kmp_hwloc_alloc_membind(hwloc_memattr_id_e attr, size_t size, @@ -1611,7 +1611,7 @@ void *__kmp_hwloc_membind_policy(omp_memspace_handle_t ms, size_t size, return NULL; #endif } -#endif // KMP_USE_HWLOC +#endif // KMP_HWLOC_ENABLED void __kmp_init_target_mem() { *(void **)(&kmp_target_alloc_host) = KMP_DLSYM("llvm_omp_target_alloc_host"); @@ -1680,13 +1680,13 @@ omp_allocator_handle_t __kmpc_init_allocator(int gtid, omp_memspace_handle_t ms, al->fb_data = RCAST(kmp_allocator_t *, traits[i].value); break; case omp_atk_partition: -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED al->membind = (omp_alloctrait_value_t)traits[i].value; KMP_DEBUG_ASSERT(al->membind == omp_atv_environment || al->membind == omp_atv_nearest || al->membind == omp_atv_blocked || al->membind == omp_atv_interleaved); -#endif +#endif // KMP_HWLOC_ENABLED al->memkind = RCAST(void **, traits[i].value); break; case omp_atk_pin_device: @@ -1980,7 +1980,7 @@ void *__kmp_alloc(int gtid, size_t algn, size_t size, } } -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED if (__kmp_hwloc_available) { if (__kmp_is_hwloc_membind_supported(HWLOC_MEMBIND_BIND)) { if (allocator < kmp_max_mem_alloc) { @@ -2074,7 +2074,7 @@ void *__kmp_alloc(int gtid, size_t algn, size_t size, ptr = hwloc_alloc(__kmp_hwloc_topology, desc.size_a); } } else { -#endif +#endif // KMP_HWLOC_ENABLED if (__kmp_memkind_available) { if (allocator < kmp_max_mem_alloc) { // pre-defined allocator @@ -2201,9 +2201,9 @@ void *__kmp_alloc(int gtid, size_t algn, size_t size, KMP_ASSERT(0); // abort fallback requested } // no sense to look for another fallback because of same internal alloc } -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED } -#endif +#endif // KMP_HWLOC_ENABLED KE_TRACE(10, ("__kmp_alloc: T#%d %p=alloc(%d)\n", gtid, ptr, desc.size_a)); if (ptr == NULL) return NULL; @@ -2339,7 +2339,7 @@ void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t allocator) { kmp_target_unlock_mem(desc.ptr_alloc, device); } -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED if (__kmp_hwloc_available) { if (oal > kmp_max_mem_alloc && al->pool_size > 0) { kmp_uint64 used = @@ -2349,7 +2349,7 @@ void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t allocator) { } hwloc_free(__kmp_hwloc_topology, desc.ptr_alloc, desc.size_a); } else { -#endif +#endif // KMP_HWLOC_ENABLED if (__kmp_memkind_available) { if (oal < kmp_max_mem_alloc) { // pre-defined allocator @@ -2378,9 +2378,9 @@ void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t allocator) { } __kmp_thread_free(__kmp_thread_from_gtid(gtid), desc.ptr_alloc); } -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED } -#endif +#endif // KMP_HWLOC_ENABLED } /* If LEAK_MEMORY is defined, __kmp_free() will *not* free memory. It causes diff --git a/openmp/runtime/src/kmp_dispatch.h b/openmp/runtime/src/kmp_dispatch.h index cf19eb52662c..f161a801700f 100644 --- a/openmp/runtime/src/kmp_dispatch.h +++ b/openmp/runtime/src/kmp_dispatch.h @@ -182,12 +182,12 @@ template <typename T> struct dispatch_shared_info_template { #if KMP_USE_HIER_SCHED kmp_hier_t<T> *hier; #endif -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED // When linking with libhwloc, the ORDERED EPCC test slowsdown on big // machines (> 48 cores). Performance analysis showed that a cache thrash // was occurring and this padding helps alleviate the problem. char padding[64]; -#endif +#endif // KMP_HWLOC_ENABLED }; /* ------------------------------------------------------------------------ */ diff --git a/openmp/runtime/src/kmp_global.cpp b/openmp/runtime/src/kmp_global.cpp index 87c0a66a16c0..f19d83bb5d05 100644 --- a/openmp/runtime/src/kmp_global.cpp +++ b/openmp/runtime/src/kmp_global.cpp @@ -250,10 +250,10 @@ enum mic_type __kmp_mic_type = non_mic; KMPAffinity *__kmp_affinity_dispatch = NULL; -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED int __kmp_hwloc_error = FALSE; hwloc_topology_t __kmp_hwloc_topology = NULL; -#endif +#endif // KMP_HWLOC_ENABLED #if KMP_OS_WINDOWS #if KMP_GROUP_AFFINITY diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp index 392a02ebbd9a..d806b02ade4d 100644 --- a/openmp/runtime/src/kmp_settings.cpp +++ b/openmp/runtime/src/kmp_settings.cpp @@ -1069,10 +1069,10 @@ static void __kmp_stg_print_warnings(kmp_str_buf_t *buffer, char const *name, static void __kmp_stg_parse_nesting_mode(char const *name, char const *value, void *data) { __kmp_stg_parse_int(name, value, 0, INT_MAX, &__kmp_nesting_mode); -#if KMP_AFFINITY_SUPPORTED && KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED if (__kmp_nesting_mode > 0) __kmp_affinity_top_method = affinity_top_method_hwloc; -#endif +#endif // KMP_HWLOC_ENABLED } // __kmp_stg_parse_nesting_mode static void __kmp_stg_print_nesting_mode(kmp_str_buf_t *buffer, @@ -3291,11 +3291,11 @@ static void __kmp_stg_parse_topology_method(char const *name, char const *value, if (__kmp_str_match("all", 1, value)) { __kmp_affinity_top_method = affinity_top_method_all; } -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED else if (__kmp_str_match("hwloc", 1, value)) { __kmp_affinity_top_method = affinity_top_method_hwloc; } -#endif +#endif // KMP_HWLOC_ENABLED #if KMP_ARCH_X86 || KMP_ARCH_X86_64 else if (__kmp_str_match("cpuid_leaf31", 12, value) || __kmp_str_match("cpuid 1f", 8, value) || @@ -3399,11 +3399,11 @@ static void __kmp_stg_print_topology_method(kmp_str_buf_t *buffer, break; #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED case affinity_top_method_hwloc: value = "hwloc"; break; -#endif +#endif // KMP_HWLOC_ENABLED case affinity_top_method_cpuinfo: value = "cpuinfo"; @@ -6289,7 +6289,7 @@ void __kmp_env_initialize(char const *string) { #if KMP_AFFINITY_SUPPORTED if (!TCR_4(__kmp_init_middle)) { -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED // Force using hwloc when either tiles or numa nodes requested within // KMP_HW_SUBSET or granularity setting and no other topology method // is requested @@ -6304,12 +6304,12 @@ void __kmp_env_initialize(char const *string) { if (__kmp_affinity.gran == KMP_HW_NUMA || __kmp_affinity.gran == KMP_HW_TILE) __kmp_affinity_top_method = affinity_top_method_hwloc; -#endif +#endif // KMP_HWLOC_ENABLED // Determine if the machine/OS is actually capable of supporting // affinity. const char *var = "KMP_AFFINITY"; KMPAffinity::pick_api(); -#if KMP_USE_HWLOC +#if KMP_HWLOC_ENABLED // If Hwloc topology discovery was requested but affinity was also disabled, // then tell user that Hwloc request is being ignored and use default // topology discovery method. @@ -6318,7 +6318,7 @@ void __kmp_env_initialize(char const *string) { KMP_WARNING(AffIgnoringHwloc, var); __kmp_affinity_top_method = affinity_top_method_all; } -#endif +#endif // KMP_HWLOC_ENABLED if (__kmp_affinity.type == affinity_disabled) { KMP_AFFINITY_DISABLE(); } else if (!KMP_AFFINITY_CAPABLE()) { |