Implement linker list

Author: rikkimax

changelog/dmd.linkerlist.dd

@@ -0,0 +1,34 @@
+Linker lists are now supported
+
+A linker list is a list of symbols that a linker puts together into a section, allowing for each disparate module to provide entries within their object files.
+They cannot be exported or dllimport, you must register their contents by use of a module constructor (can be C).
+
+To iterate them, you may foreach over them at runtime.
+
+To append to them, you may do so from within a CTFE executed function.
+
+```d
+import core.attribute;
+
+__gshared __linkerlist!int myInts;
+
+static assert(() { myInts ~= 42; return true; }());
+
+void main() {
+    foreach(v; myInts)
+        assert(v == 42);
+}
+```
+
+Each declaration for the linker list, must be per executable or shared library.
+The declaration may trigger symbol emittance.
+
+If the type is ``const`` or ``immutable`` then the list entries will be marked as read only.
+
+An entry when emitted will go into the object file, associated with the module that initiated the function.
+It is recommended therefore that the function should always be templated.
+If the template is associated with a module that is not being compiled (imported), it will not be emitted into an object file.
+Note: that this module does NOT refer to the source file that contains the template.
+
+If you are already using linker lists using the section attribute, it is strongly recommended to swap to compiler backed linker lists.
+They are platform, linker, and compiler specific abstractions that are not portable.

compiler/src/dmd/declaration.d

@@ -427,15 +427,17 @@ extern (C++) final class OverDeclaration : Declaration
 extern (C++) class VarDeclaration : Declaration
 {
     Initializer _init;
-    FuncDeclarations nestedrefs;    // referenced by these lexically nested functions
-    TupleDeclaration aliasTuple;    // when `this` is really a tuple of declarations
-    VarDeclaration lastVar;         // Linked list of variables for goto-skips-init detection
-    Expression edtor;               // if !=null, does the destruction of the variable
-    IntRange* range;                // if !=null, the variable is known to be within the range
+    FuncDeclarations nestedrefs;       // referenced by these lexically nested functions
+    TupleDeclaration aliasTuple;       // when `this` is really a tuple of declarations
+    VarDeclaration lastVar;            // Linked list of variables for goto-skips-init detection
+    Expression edtor;                  // if !=null, does the destruction of the variable
+    IntRange* range;                   // if !=null, the variable is known to be within the range
 
-    uint endlinnum;                 // line number of end of scope that this var lives in
+    VarDeclaration entryForLinkerList; // if !=null, the linker list we're appending to
+
+    uint endlinnum;                    // line number of end of scope that this var lives in
     uint offset;
-    uint sequenceNumber;            // order the variables are declared
+    uint sequenceNumber;               // order the variables are declared
     structalign_t alignment;
 
     // When interpreting, these point to the value (NULL if value not determinable)
@@ -469,6 +471,8 @@ extern (C++) class VarDeclaration : Declaration
         bool dllExport;         /// __declspec(dllexport)
         mixin VarDeclarationExtra;
         bool systemInferred;    /// @system was inferred from initializer
+
+        bool isLinkerListDeclaration; /// This variable declares a linker list
     }
 
     import dmd.common.bitfields : generateBitFields;

compiler/src/dmd/declaration.h

@@ -236,6 +236,7 @@ class VarDeclaration : public Declaration
     VarDeclaration *lastVar;    // Linked list of variables for goto-skips-init detection
     Expression *edtor;          // if !=NULL, does the destruction of the variable
     IntRange *range;            // if !NULL, the variable is known to be within the range
+    VarDeclaration* entryForLinkerList;
 
     unsigned endlinnum;         // line number of end of scope that this var lives in
     unsigned offset;
@@ -282,6 +283,8 @@ class VarDeclaration : public Declaration
 #endif
     bool systemInferred() const;
     bool systemInferred(bool v);
+    bool isLinkerListDeclaration() const;
+    bool isLinkerListDeclaration(bool v);
     static VarDeclaration *create(Loc loc, Type *t, Identifier *id, Initializer *init, StorageClass storage_class = STCundefined);
     VarDeclaration *syntaxCopy(Dsymbol *) override;
     const char *kind() const override;

compiler/src/dmd/dinterpret.d

@@ -4652,6 +4652,47 @@ public:
         Expression pthis = null;
         FuncDeclaration fd = null;
 
+        if (e.f !is null && e.f.ident is Id.opOpAssign)
+        {
+            DotVarExp dve = e.e1.isDotVarExp();
+            VarExp ve;
+            VarDeclaration vd, linkerListEntry;
+
+            if (dve !is null &&
+                (ve = dve.e1.isVarExp()) !is null &&
+                (vd = ve.var.isVarDeclaration()) !is null &&
+                vd.isLinkerListDeclaration)
+            {
+                if (istate.fd is null)
+                {
+                    error(e.loc, "cannot append to a linker list without being in a function");
+                    result = CTFEExp.cantexp;
+                }
+                else
+                {
+                    auto root = istate.fd.getModule;
+                    // If we don't have a root module, to put it in we'll ignore it.
+                    // We ignore it because it may have already been emitted in a different compilation step.
+
+                    if (root.isRoot)
+                    {
+                        auto initExp = interpret((*e.arguments)[0], istate).copyRegionExp;
+                        auto init = new ExpInitializer(e.loc, initExp);
+                        linkerListEntry = new VarDeclaration(e.loc, null, Identifier.generateIdWithLoc("_d_linkerlistentry", e.loc), init);
+
+                        linkerListEntry.entryForLinkerList = vd;
+                        linkerListEntry.visibility.kind = Visibility.Kind.private_;
+                        linkerListEntry.storage_class |= STC.gshared;
+
+                        root.members.push(linkerListEntry);
+                        linkerListEntry.dsymbolSemantic(root._scope);
+                    }
+                }
+
+                return;
+            }
+        }
+
         Expression ecall = interpretRegion(e.e1, istate);
         if (exceptionOrCant(ecall))
             return;

compiler/src/dmd/dsymbolsem.d

@@ -2592,6 +2592,26 @@ private extern(C++) final class DsymbolSemanticVisitor : Visitor
                 // Flag variable as error to avoid invalid error messages due to unknown size
                 dsym.type = Type.terror;
             }
+            else if (ts.sym.ident is Id.__linkerlist)
+            {
+                // Okay we've got a linker list
+
+                if (dsym.visibility.kind == Visibility.Kind.export_)
+                {
+                    .error(dsym.loc, "Linker lists cannot be exported");
+                    dsym.type = Type.terror;
+                }
+                else if (dsym.storage_class & STC.extern_)
+                {
+                    .error(dsym.loc, "Linker lists cannot be `extern` they must live at declaration site");
+                    dsym.type = Type.terror;
+                }
+                else
+                {
+                    dsym.isLinkerListDeclaration = true;
+                    dsym._linkage = LINK.system; // set mangling to system, as we can end up going too big with D.
+                }
+            }
         }
         if ((dsym.storage_class & STC.auto_) && !inferred && !(dsym.storage_class & STC.autoref))
             .error(dsym.loc, "%s `%s` - storage class `auto` has no effect if type is not inferred, did you mean `scope`?", dsym.kind, dsym.toPrettyChars);

compiler/src/dmd/frontend.h

@@ -6598,6 +6598,7 @@ class VarDeclaration : public Declaration
     VarDeclaration* lastVar;
     Expression* edtor;
     IntRange* range;
+    VarDeclaration* entryForLinkerList;
     uint32_t endlinnum;
     uint32_t offset;
     uint32_t sequenceNumber;
@@ -6639,6 +6640,8 @@ class VarDeclaration : public Declaration
     bool inAlignSection(bool v);
     bool systemInferred() const;
     bool systemInferred(bool v);
+    bool isLinkerListDeclaration() const;
+    bool isLinkerListDeclaration(bool v);
 private:
     uint32_t bitFields;
 public:
@@ -8668,6 +8671,7 @@ struct Id final
     static Identifier* classDelete;
     static Identifier* apply;
     static Identifier* applyReverse;
+    static Identifier* linkerlistApply;
     static Identifier* Fempty;
     static Identifier* Ffront;
     static Identifier* Fback;

compiler/src/dmd/glue/e2ir.d

@@ -21,6 +21,7 @@ import dmd.root.ctfloat;
 import dmd.root.rmem;
 import dmd.rootobject;
 import dmd.root.stringtable;
+import dmd.common.outbuffer;
 
 import dmd.glue;
 import dmd.glue.objc;
@@ -56,6 +57,8 @@ import dmd.id;
 import dmd.init;
 import dmd.location;
 import dmd.mtype;
+import dmd.mangle : mangleToBuffer;
+import dmd.opover : search_function;
 import dmd.printast;
 import dmd.sideeffect;
 import dmd.statement;
@@ -5408,7 +5411,7 @@ elem* callfunc(Loc loc,
     version (none)
     {
         printf("callfunc(directcall = %d, tret = '%s', ec = %p, fd = %p, op = %d)\n",
-            directcall, tret.toChars(), ec, fd);
+            directcall, tret.toChars(), ec, fd, op);
         printf("ec: "); elem_print(ec);
         if (fd)
             printf("fd = '%s', vtblIndex = %d, isVirtual() = %d\n", fd.toChars(), fd.vtblIndex, fd.isVirtual());
@@ -5521,7 +5524,7 @@ elem* callfunc(Loc loc,
 
         /* Convert arguments[] to elems[] in left-to-right order
          */
-        const n = arguments.length;
+        const n = arguments.length + (fd !is null && fd.ident is Id.linkerlistApply);
         debug
             elem*[2] elems_array = void;
         else
@@ -5538,8 +5541,65 @@ elem* callfunc(Loc loc,
         const int j = tf.isDstyleVariadic();
         const osx_aapcs64 = irs.target.isAArch64 && irs.target.os == Target.OS.OSX;
 
+        void linkerlistArg(Expression arg)
+        {
+            // We are swapping a __linkerlist!T global variable with its respective start+stop symbols as arguments.
+            // The transformation takes place at offset 1 of the elems array, and will result in three arguments total.
+
+            VarExp ve = arg.isVarExp;
+            assert(ve !is null);
+            VarDeclaration vd  = ve.var.isVarDeclaration;
+            assert(vd !is null);
+            const canBeReadOnly = !vd.type.isMutable;
+
+            elem* genStartEndExtern(string namePrefix, string nameSuffix, bool noUnderscore = false)
+            {
+                OutBuffer nameOB;
+                nameOB.writestring(namePrefix);
+                mangleToBuffer(vd, nameOB);
+                nameOB.writestring(nameSuffix);
+
+                Symbol* s = symbol_calloc(nameOB.extractSlice);
+                s.Sclass = SC.extern_;
+                s.Sfl = FL.extern_;
+                s.Stype = type_fake(TYnptr); // always a pointer in size, regardless of linker list type.
+
+                if (noUnderscore)
+                    s.Sflags |= SFLnounderscore;
+
+                return el_ptr(s);
+            }
+
+            switch (config.objfmt)
+            {
+                case OBJ_MACH:
+                    elems[1] = genStartEndExtern(canBeReadOnly ? "section$start$__TEXT$" : "section$start$__DATA$", "", true);
+                    elems[2] = genStartEndExtern(canBeReadOnly ? "section$end$__TEXT$" : "section$end$__DATA$", "", true);
+                    break;
+
+                case OBJ_ELF:
+                    elems[1] = genStartEndExtern("__start_", "");
+                    elems[2] = genStartEndExtern("__stop_", "");
+                    break;
+                case OBJ_MSCOFF:
+                    elems[1] = genStartEndExtern("", "_start");
+                    elems[2] = genStartEndExtern("", "_end");
+                    break;
+
+                default:
+                    assert(0);
+            }
+        }
+
         foreach (const i, arg; *arguments)
         {
+            if (i == 1 && fd !is null && fd.ident is Id.linkerlistApply)
+            {
+                // do not generate symbol for the linker list global
+                linkerlistArg(arg);
+                break; // This completes the argument list.
+            }
+
             elem* ea = toElem(arg, irs);
             Parameter param = null;
 
@@ -6555,8 +6615,10 @@ elem* toElemCall(CallExp ce, ref IRState irs, elem* ehidden = null)
     if (ce.e1.op == EXP.dotVariable && t1.ty != Tdelegate)
     {
         DotVarExp dve = cast(DotVarExp)ce.e1;
+        FuncDeclaration linkerListApply;
 
         fd = dve.var.isFuncDeclaration();
+        ectype = dve.e1.type.toBasetype();
 
         if (auto sle = dve.e1.isStructLiteralExp())
         {
@@ -6565,6 +6627,17 @@ elem* toElemCall(CallExp ce, ref IRState irs, elem* ehidden = null)
                 sle.type.size() <= 8)          // more efficient than fPIC
                 sle.useStaticInit = false;     // don't modify initializer, so make copy
         }
+        else if (fd.ident is Id.apply)
+        {
+            if (auto ts = ectype.isTypeStruct)
+            {
+                if (ts.sym.ident is Id.__linkerlist)
+                {
+                    Dsymbol applyImpl = search_function(ts.sym, Id.linkerlistApply);
+                    linkerListApply = applyImpl !is null ? applyImpl.isFuncDeclaration : null;
+                }
+            }
+        }
 
         // Special cases for constructor RVO
         if (ehidden && fd && fd.isCtorDeclaration())
@@ -6584,11 +6657,22 @@ elem* toElemCall(CallExp ce, ref IRState irs, elem* ehidden = null)
 
             ehidden = null;
         }
+        else if (linkerListApply !is null)
+        {
+            // Point ec to the implementation function and update the function/type to match.
+            // Append to the arguments list the global variable that contains the linker list.
+
+            ec = el_ptr(toSymbol(linkerListApply));
+            ectype = linkerListApply.type;
+            t1 = linkerListApply.type;
+            fd = linkerListApply;
+
+            assert(ce.arguments !is null && ce.arguments.length == 1);
+            ce.arguments.push(dve.e1);
+        }
         else
             ec = toElem(dve.e1, irs);
 
-        ectype = dve.e1.type.toBasetype();
-
         /* Recognize:
          *   [1] ce:  ((S __ctmp = initializer),__ctmp).ctor(args)
          * where the left of the . was turned into [2] or [3] for EH_DWARF:

compiler/src/dmd/glue/tocsym.d

@@ -139,6 +139,11 @@ Symbol* toSymbol(Dsymbol s)
                 fprintf(stderr, "VarDeclaration.toSymbol(%s) needThis kind: %s\n", vd.toPrettyChars(), vd.kind());
             assert(!vd.needThis());
 
+            // foreach is already handled before this, so anything thats hitting it is an error case.
+            // No need to quit after error, at worse this will result in a linker error.
+            if (vd.isLinkerListDeclaration)
+                error(vd.loc, "Linker list declarations cannot be accessed except for iteration via foreach statement or appended during CTFE");
+
             import dmd.common.outbuffer : OutBuffer;
             OutBuffer buf;
             bool isNRVO = false;

compiler/src/dmd/glue/todt.d

@@ -1486,7 +1486,7 @@ private extern (C++) class TypeInfoDtVisitor : Visitor
         auto tc = d.tinfo.isTypeStruct();
         StructDeclaration sd = tc.sym;
 
-        if (!sd.members)
+        if (!sd.members || sd.ident is Id.__linkerlist)
             return;
 
         if (sd.semanticRun < PASS.semantic3done)