schema: so much boilerplate for feeding information to the Compiler that I...

schema: so much boilerplate for feeding information to the Compiler that I wrote another supplementary code generator.

(I'm getting very weary of golang.)

This new bit of codegen makes the compiler.go file fairly readable
again, though, so I'm satisfied with it.

The Compiler API is now complete enough that I can start repairing
other things to use it properly.  The schemadmt.Schema.Compile()
function and all of its helpers compile again now.  So does *most*
of the whole codegen system... with the notable exception of all
the hardcoded typesystem spawning which used the old placeholder
methods which have now been stricken.

TypeSystem now maintains order.  This allowed me to remove some
sort operations from the code generator.  This also means the next
time any existing codegen is re-run, the output file will shift
significantly.  However, it shouldn't do so again in the future.

schema/compiler.go

@@ -53,15 +53,19 @@ type Compiler struct {
 	//  At the end of the day, though?  Honestly, import cycle breaking.  This was not the first choice.
 	// An implementation which wraps the schemadmt package to make it fit the schema interfaces was the first choice
 	//  because it would've saved a *lot* of work (it would've removed the need for this compiler system entirely, among other things);
-	//  but that doesn't fly, because the dmt types have to implement Type, and that interface refers to yet more * types.
+	//  but that doesn't fly, because the generated nodes have to implement schema.TypedNode, and that interface refers to other types like schema.Type
+	//   (and even if all of those are interfaces, you still can't have conformational-equality span multiple interface types in golang).
 	//  And that would make an import cycle if we tried to put types wrapping the dmt types into the schema package.  Whoops.
 	// A separation of this feature into its own "compiler" package was also attempted; this too did not work out well.
+	//  (The main reason to desire this would be clarity and grouping.  The reading interfaces and the creation stuff are for different audiences.)
 	//  We want immutability in almost all of these values, and in golang, the only way to approach immutability is with unexported symbols and package boundaries;
 	//  there aren't many practical approaches which would allow separating the reading and the creating parts of handling the same structure into two different packages.
-	//  One approach is to simply define all reading via interfaces.  This works and is not wildly unusual in golang.
+	//  One approach that does allow such a split is to simply define all reading via interfaces.  This works and is not wildly unusual in golang.
 	//   However, to apply this in practice in this case would result in many interfaces which are just shells around exactly one implementation;
 	//   and it would also suggest various implementations are expected, which... is simply not the case.
+	//   This would still overall be viable, but the result would be pushing a bit into "strange".
 	// So, here we are.
+	//  Compiler is a type in the schema package.  And we've attempted to attach *all* operations relating to creating schema data to it for purposes of grouping and clarity.
 
 	// ts gathers all the in-progress types (including anonymous ones),
 	// and is eventually the value we return (if Compile is ultimately successful).
@@ -76,6 +80,7 @@ func (c *Compiler) Init() {
 	c.ts = &TypeSystem{
 		map[TypeReference]Type{},
 		nil,
+		nil,
 	}
 }
 
@@ -98,6 +103,8 @@ func (c *Compiler) mustHaveNameFree(name TypeName) {
 	}
 }
 
+//go:generate sed -i /---/q compiler_carriers.go
+
 func (c *Compiler) TypeBool(name TypeName) {
 	c.addType(&TypeBool{c.ts, name})
 }
@@ -141,38 +148,17 @@ func (c *Compiler) TypeStruct(name TypeName, fields structFieldList, rstrat Stru
 	}
 }
 
-// structFieldList is a carrier type that just wraps a slice reference.
-// It is used so we can let code outside this package hold a value of this type without letting the slice become mutable.
-type structFieldList struct {
-	x []StructField
-}
+//go:generate quickimmut -output=compiler_carriers.go -attach=Compiler list StructField
 
-func (Compiler) MakeStructFieldList(fields ...StructField) structFieldList {
-	return structFieldList{fields}
-}
 func (Compiler) MakeStructField(name StructFieldName, typ TypeReference, optional, nullable bool) StructField {
 	return StructField{nil, name, typ, optional, nullable}
 }
 
-func MakeStructRepresentation_Map(fieldDetails ...StructRepresentation_Map_FieldDetailsEntry) StructRepresentation {
-	rstrat := StructRepresentation_Map{nil, make(map[StructFieldName]StructRepresentation_Map_FieldDetails, len(fieldDetails))}
-	for _, fd := range fieldDetails {
-		if _, exists := rstrat.fieldDetails[fd.FieldName]; exists {
-			panic(fmt.Errorf("field name %q duplicated", fd.FieldName))
-		}
-		rstrat.fieldDetails[fd.FieldName] = fd.Details
-	}
-	return rstrat
+func (Compiler) MakeStructRepresentation_Map(fieldDetails structFieldNameStructRepresentation_Map_FieldDetailsMap) StructRepresentation {
+	return StructRepresentation_Map{nil, fieldDetails.x}
 }
 
-// StructRepresentation_Map_FieldDetailsEntry is a carrier type that associates a field name
-// with field detail information that's appropriate to a map representation strategy for a struct.
-// It is used to feed data to MakeStructRepresentation_Map in so that that method can build a map
-// without exposing a reference to it in a way that would make that map mutable.
-type StructRepresentation_Map_FieldDetailsEntry struct {
-	FieldName StructFieldName
-	Details   StructRepresentation_Map_FieldDetails
-}
+//go:generate quickimmut -output=compiler_carriers.go -attach=Compiler map StructFieldName StructRepresentation_Map_FieldDetails
 
 func (c *Compiler) TypeMap(name TypeName, keyTypeRef TypeName, valueTypeRef TypeReference, valueNullable bool) {
 	c.addType(&TypeMap{c.ts, name, keyTypeRef, valueTypeRef, valueNullable})
@@ -182,7 +168,7 @@ func (c *Compiler) TypeList(name TypeName, valueTypeRef TypeReference, valueNull
 	c.addType(&TypeList{c.ts, name, valueTypeRef, valueNullable})
 }
 
-func (c *Compiler) TypeUnion(name TypeName, members unionMemberList, rstrat UnionRepresentation) {
+func (c *Compiler) TypeUnion(name TypeName, members typeNameList, rstrat UnionRepresentation) {
 	t := TypeUnion{
 		ts:      c.ts,
 		name:    name,
@@ -191,45 +177,14 @@ func (c *Compiler) TypeUnion(name TypeName, members unionMemberList, rstrat Unio
 	}
 	c.addType(&t)
 	// note! duplicate member names *not* rejected at this moment -- that's a job for the validation phase.
-	//  this is an interesting contrast to how when buildings struct, dupe field names may be rejected proactively:
+	//  this is an interesting contrast to how when building structs, dupe field names may be rejected proactively:
 	//   the difference is, member names were a list in the dmt form too, so it's important we format a nice error rather than panic if there was invalid data there.
 }
 
-// unionMemberList is a carrier type that just wraps a slice reference.
-// It is used so we can let code outside this package hold a value of this type without letting the slice become mutable.
-type unionMemberList struct {
-	x []TypeName
-}
+//go:generate quickimmut -output=compiler_carriers.go -attach=Compiler list TypeName
 
-func (Compiler) MakeUnionMemberList(members ...TypeName) unionMemberList {
-	return unionMemberList{members}
-}
-
-func (Compiler) MakeUnionRepresentation_Keyed(discriminantTable unionDiscriminantStringTable) UnionRepresentation {
+func (Compiler) MakeUnionRepresentation_Keyed(discriminantTable stringTypeNameMap) UnionRepresentation {
 	return &UnionRepresentation_Keyed{nil, discriminantTable.x}
 }
 
-// unionMemberList is a carrier type that just wraps a map reference.
-// It is used so we can let code outside this package hold a value of this type without letting the map become mutable.
-type unionDiscriminantStringTable struct {
-	x map[string]TypeName
-}
-
-func (Compiler) MakeUnionDiscriminantStringTable(entries ...UnionDiscriminantStringEntry) unionDiscriminantStringTable {
-	x := make(map[string]TypeName, len(entries))
-	for _, y := range entries {
-		if _, exists := x[y.Discriminant]; exists {
-			panic(fmt.Errorf("discriminant string %q duplicated", y.Discriminant))
-		}
-		x[y.Discriminant] = y.Member
-	}
-	return unionDiscriminantStringTable{x}
-}
-
-// UnionRepresentation_DiscriminantStringEntry is a carrier type that associates a string with a TypeName.
-// It is used to feed data to several of the union representation constructors so that those functions
-// can build their results without exposing a reference to a map in a way that would make that map mutable.
-type UnionDiscriminantStringEntry struct {
-	Discriminant string
-	Member       TypeName
-}
+//go:generate quickimmut -output=compiler_carriers.go -attach=Compiler map string TypeName

schema/compiler_carriers.go

+package schema
+
+// This file is for types and functions that aren't part of the readable API of this package,
+// but were needed to make some part of the Compiler's API expressable.
+// I call this file "carriers" because all these types are just short-lived carriers for data,
+// and their content ends up remunged and handled by the Compiler almost immediately after creation.
+//
+// Coincidentally, they're all so boring that I templated them.
+//
+// (Nothing in this file would be necessary if we had been able to use the schemadmt types directly,
+// since they would've provided all the semantics these types (re)describe;
+// these very boring types are only necessary because we are in a unique position
+// of being unable to use our own codegen due for import cycle reasons (see other comments in Compiler about this).)
+
+// Code generated by comments in carrier.go.  DO NOT EDIT below this line.
+// ---
+
+type structFieldList struct {
+	x []StructField
+}
+type structFieldListBuilder structFieldList
+
+func (Compiler) MakeStructFieldList(ents ...StructField) structFieldList {
+	x := make([]StructField, len(ents))
+	copy(x, ents)
+	return structFieldList{x}
+}
+func (Compiler) StartStructFieldList(sizeHint int) structFieldListBuilder {
+	return structFieldListBuilder{make([]StructField, 0, sizeHint)}
+}
+func (b *structFieldListBuilder) Append(v StructField) {
+	b.x = append(b.x, v)
+}
+func (b *structFieldListBuilder) Finish() structFieldList {
+	v := *b
+	b.x = nil
+	return structFieldList(v)
+}
+
+type structFieldNameStructRepresentation_Map_FieldDetailsMap struct {
+	x map[StructFieldName]StructRepresentation_Map_FieldDetails
+}
+type structFieldNameStructRepresentation_Map_FieldDetailsEntry struct {
+	k StructFieldName
+	v StructRepresentation_Map_FieldDetails
+}
+type structFieldNameStructRepresentation_Map_FieldDetailsMapBuilder structFieldNameStructRepresentation_Map_FieldDetailsMap
+
+func (Compiler) MakeStructFieldNameStructRepresentation_Map_FieldDetailsMap(ents ...structFieldNameStructRepresentation_Map_FieldDetailsEntry) structFieldNameStructRepresentation_Map_FieldDetailsMap {
+	x := make(map[StructFieldName]StructRepresentation_Map_FieldDetails, len(ents))
+	for _, y := range ents {
+		x[y.k] = y.v
+	}
+	return structFieldNameStructRepresentation_Map_FieldDetailsMap{x}
+}
+func (Compiler) MakeStructFieldNameStructRepresentation_Map_FieldDetailsMapEntry(k StructFieldName, v StructRepresentation_Map_FieldDetails) structFieldNameStructRepresentation_Map_FieldDetailsEntry {
+	return structFieldNameStructRepresentation_Map_FieldDetailsEntry{k, v}
+}
+func (Compiler) StartStructFieldNameStructRepresentation_Map_FieldDetailsMap(sizeHint int) structFieldNameStructRepresentation_Map_FieldDetailsMapBuilder {
+	return structFieldNameStructRepresentation_Map_FieldDetailsMapBuilder{make(map[StructFieldName]StructRepresentation_Map_FieldDetails, sizeHint)}
+}
+func (b *structFieldNameStructRepresentation_Map_FieldDetailsMapBuilder) Append(k StructFieldName, v StructRepresentation_Map_FieldDetails) {
+	b.x[k] = v
+}
+func (b *structFieldNameStructRepresentation_Map_FieldDetailsMapBuilder) Finish() structFieldNameStructRepresentation_Map_FieldDetailsMap {
+	v := *b
+	b.x = nil
+	return structFieldNameStructRepresentation_Map_FieldDetailsMap(v)
+}
+
+type typeNameList struct {
+	x []TypeName
+}
+type typeNameListBuilder typeNameList
+
+func (Compiler) MakeTypeNameList(ents ...TypeName) typeNameList {
+	x := make([]TypeName, len(ents))
+	copy(x, ents)
+	return typeNameList{x}
+}
+func (Compiler) StartTypeNameList(sizeHint int) typeNameListBuilder {
+	return typeNameListBuilder{make([]TypeName, 0, sizeHint)}
+}
+func (b *typeNameListBuilder) Append(v TypeName) {
+	b.x = append(b.x, v)
+}
+func (b *typeNameListBuilder) Finish() typeNameList {
+	v := *b
+	b.x = nil
+	return typeNameList(v)
+}
+
+type stringTypeNameMap struct {
+	x map[string]TypeName
+}
+type stringTypeNameEntry struct {
+	k string
+	v TypeName
+}
+type stringTypeNameMapBuilder stringTypeNameMap
+
+func (Compiler) MakeStringTypeNameMap(ents ...stringTypeNameEntry) stringTypeNameMap {
+	x := make(map[string]TypeName, len(ents))
+	for _, y := range ents {
+		x[y.k] = y.v
+	}
+	return stringTypeNameMap{x}
+}
+func (Compiler) MakeStringTypeNameMapEntry(k string, v TypeName) stringTypeNameEntry {
+	return stringTypeNameEntry{k, v}
+}
+func (Compiler) StartStringTypeNameMap(sizeHint int) stringTypeNameMapBuilder {
+	return stringTypeNameMapBuilder{make(map[string]TypeName, sizeHint)}
+}
+func (b *stringTypeNameMapBuilder) Append(k string, v TypeName) {
+	b.x[k] = v
+}
+func (b *stringTypeNameMapBuilder) Finish() stringTypeNameMap {
+	v := *b
+	b.x = nil
+	return stringTypeNameMap(v)
+}

schema/dmt/compile.go

@@ -116,7 +116,7 @@ func (schdmt Schema) Compile() (schema.TypeSystem, error) {
 			// Feed it all into the compiler.
 			c.TypeUnion(
 				schema.TypeName(tn.String()),
-				schema.Compiler{}.MakeUnionMemberList(members...),
+				schema.Compiler{}.MakeTypeNameList(members...),
 				rstrat,
 			)
 		case TypeEnum:
@@ -141,14 +141,14 @@ func (dmt TypeNameOrInlineDefn) compile(c *schema.Compiler) {
 
 func (dmt StructRepresentation_Map) compile() schema.StructRepresentation {
 	if !dmt.FieldFields().Exists() {
-		return schema.MakeStructRepresentation_Map()
+		return schema.Compiler{}.MakeStructRepresentation_Map(schema.Compiler{}.MakeStructFieldNameStructRepresentation_Map_FieldDetailsMap())
 	}
-	fields := make([]schema.StructRepresentation_Map_FieldDetailsEntry, dmt.FieldFields().Must().Length())
+	fields := schema.Compiler{}.StartStructFieldNameStructRepresentation_Map_FieldDetailsMap(int(dmt.FieldFields().Must().Length()))
 	for itr := dmt.FieldFields().Must().Iterator(); !itr.Done(); {
 		fn, det := itr.Next()
-		fields = append(fields, schema.StructRepresentation_Map_FieldDetailsEntry{
-			FieldName: schema.StructFieldName(fn.String()),
-			Details: schema.StructRepresentation_Map_FieldDetails{
+		fields.Append(
+			schema.StructFieldName(fn.String()),
+			schema.StructRepresentation_Map_FieldDetails{
 				Rename: func() string {
 					if det.FieldRename().Exists() {
 						return det.FieldRename().Must().String()
@@ -157,9 +157,9 @@ func (dmt StructRepresentation_Map) compile() schema.StructRepresentation {
 				}(),
 				Implicit: nil, // TODO
 			},
-		})
+		)
 	}
-	return schema.MakeStructRepresentation_Map(fields...)
+	return schema.Compiler{}.MakeStructRepresentation_Map(fields.Finish())
 }
 
 func (dmt StructRepresentation_Tuple) compile() schema.StructRepresentation {
@@ -179,12 +179,12 @@ func (dmt StructRepresentation_Listpairs) compile() schema.StructRepresentation
 }
 
 func (dmt UnionRepresentation_Keyed) compile() schema.UnionRepresentation {
-	ents := make([]schema.UnionDiscriminantStringEntry, 0, dmt.Length())
+	ents := schema.Compiler{}.StartStringTypeNameMap(int(dmt.Length()))
 	for itr := dmt.Iterator(); !itr.Done(); {
 		k, v := itr.Next()
-		ents = append(ents, schema.UnionDiscriminantStringEntry{k.String(), schema.TypeName(v.String())})
+		ents.Append(k.String(), schema.TypeName(v.String()))
 	}
-	return schema.Compiler{}.MakeUnionRepresentation_Keyed(schema.Compiler{}.MakeUnionDiscriminantStringTable(ents...))
+	return schema.Compiler{}.MakeUnionRepresentation_Keyed(ents.Finish())
 }
 
 func (dmt UnionRepresentation_Kinded) compile() schema.UnionRepresentation {

schema/gen/go/adjunctCfg.go

@@ -15,7 +15,7 @@ import (
 
 type FieldTuple struct {
 	TypeName  schema.TypeName
-	FieldName string
+	FieldName schema.StructFieldName
 }
 
 type AdjunctCfg struct {
@@ -55,14 +55,14 @@ func (cfg *AdjunctCfg) FieldSymbolLower(f schema.StructField) string {
 	if x, ok := cfg.FieldSymbolLowerOverrides[FieldTuple{f.Parent().Name(), f.Name()}]; ok {
 		return x
 	}
-	return f.Name() // presumed already lower
+	return string(f.Name()) // presumed already lower
 }
 
 func (cfg *AdjunctCfg) FieldSymbolUpper(f schema.StructField) string {
 	if x, ok := cfg.fieldSymbolUpperOverrides[FieldTuple{f.Type().Name(), f.Name()}]; ok {
 		return x
 	}
-	return strings.Title(f.Name())
+	return strings.Title(string(f.Name()))
 }
 
 // Comments returns a bool for whether comments should be included in gen output or not.

schema/gen/go/generate.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"sort"
 
 	"github.com/ipld/go-ipld-prime/schema"
 )
@@ -29,18 +28,12 @@ func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctC
 	// Local helper function for applying generation logic to each type.
 	//  We will end up doing this more than once because in this layout, more than one file contains part of the story for each type.
 	applyToEachType := func(fn func(tg TypeGenerator, w io.Writer), f io.Writer) {
-		// Sort the type names so we have a determinisic order; this affects output consistency.
-		//  Any stable order would do, but we don't presently have one, so a sort is necessary.
-		types := ts.GetTypes()
-		keys := make(sortableTypeNames, 0, len(types))
-		for tn := range types {
-			if _, exists := externs[tn.String()]; !exists {
-				keys = append(keys, tn)
+		types := ts.AllTypes()
+		for _, typ := range types {
+			if _, exists := externs[string(typ.Name())]; exists {
+				continue
 			}
-		}
-		sort.Sort(keys)
-		for _, tn := range keys {
-			switch t2 := types[tn].(type) {
+			switch t2 := typ.(type) {
 			case *schema.TypeBool:
 				fn(NewBoolReprBoolGenerator(pkgName, t2, adjCfg), f)
 			case *schema.TypeInt:

schema/gen/go/generators.go

@@ -177,5 +177,5 @@ func EmitTypeTable(pkgName string, ts schema.TypeSystem, adjCfg *AdjunctCfg, w i
 			{{ .Name }}__Repr _{{ . | TypeSymbol }}__ReprPrototype
 			{{- end}}
 		}
-	`, w, adjCfg, ts.GetTypes())
+	`, w, adjCfg, ts.AllTypes())
 }

schema/gen/go/testEngine_test.go

@@ -69,7 +69,7 @@ func genAndCompileAndTest(
 					return nil
 				}
 			}
-		`, w, adjCfg, ts.GetTypes())
+		`, w, adjCfg, ts.AllTypes())
 	})
 
 	// Build the genned code.

schema/type_link.go

@@ -33,15 +33,15 @@ func (t TypeLink) RepresentationBehavior() ipld.Kind {
 // -- specific to TypeLink -->
 
 // HasExpectedType returns true if the link has a hint about the type it references.
-func (t *TypeLink) HasExpectedType() bool {
+func (t *TypeLink) HasReferencedType() bool {
 	return t.expectedTypeRef != ""
 }
 
 // ExpectedType returns the type which is expected for the node on the other side of the link.
 // Nil is returned if there is no information about the expected type
 // (which may be interpreted as "any").
-func (t *TypeLink) ExpectedType() Type {
-	if !t.HasExpectedType() {
+func (t *TypeLink) ReferencedType() Type {
+	if !t.HasReferencedType() {
 		return nil
 	}
 	return t.ts.types[TypeReference(t.expectedTypeRef)]

schema/type_union.go

@@ -17,8 +17,8 @@ func (UnionRepresentation_Keyed) _UnionRepresentation()        {}
 func (UnionRepresentation_Kinded) _UnionRepresentation()       {}
 func (UnionRepresentation_Envelope) _UnionRepresentation()     {}
 func (UnionRepresentation_Inline) _UnionRepresentation()       {}
-func (UnionRepresentation_StringPrefix) _UnionRepresentation() {}
-func (UnionRepresentation_BytePrefix) _UnionRepresentation()   {}
+func (UnionRepresentation_Stringprefix) _UnionRepresentation() {}
+func (UnionRepresentation_Byteprefix) _UnionRepresentation()   {}
 
 type UnionRepresentation_Keyed struct {
 	ts                *TypeSystem
@@ -39,11 +39,11 @@ type UnionRepresentation_Inline struct {
 	discriminantKey   string
 	discriminantTable map[string]TypeName
 }
-type UnionRepresentation_StringPrefix struct {
+type UnionRepresentation_Stringprefix struct {
 	ts                *TypeSystem
 	discriminantTable map[string]TypeName
 }
-type UnionRepresentation_BytePrefix struct {
+type UnionRepresentation_Byteprefix struct {
 	ts                *TypeSystem
 	discriminantTable map[string]TypeName
 }
@@ -74,9 +74,9 @@ func (t *TypeUnion) RepresentationBehavior() ipld.Kind {
 		return ipld.Kind_Map
 	case UnionRepresentation_Inline:
 		return ipld.Kind_Map
-	case UnionRepresentation_StringPrefix:
+	case UnionRepresentation_Stringprefix:
 		return ipld.Kind_String
-	case UnionRepresentation_BytePrefix:
+	case UnionRepresentation_Byteprefix:
 		return ipld.Kind_Bytes
 	default:
 		panic("unreachable")

schema/typesystem.go

@@ -10,4 +10,22 @@ type TypeSystem struct {
 	// This is kept so we can do any listing in the order the user expects,
 	// report any errors during rule validation in the same order as the input, etc.
 	list []Type
+
+	// List of anonymous types, in sorted order simply going by the reference as a string.
+	anonTypes []Type
+}
+
+// AllTypes returns a slice of every Type in this TypeSystem.
+// This slice includes both named types and anonymous types.
+// For named types, the order in which they were specified
+// then the TypeSystem was compiled is preserved.
+func (ts *TypeSystem) AllTypes() []Type {
+	v := make([]Type, len(ts.types))
+	copy(ts.list, v)
+	copy(ts.anonTypes, v[len(ts.list):])
+	return v
+}
+
+func (ts *TypeSystem) GetType(name TypeReference) Type {
+	return ts.types[name]
 }