Simplified executable file formats

src/asm/Finalize.go

@@ -8,7 +8,7 @@ import (
 
 	"git.akyoto.dev/cli/q/src/arch/x64"
 	"git.akyoto.dev/cli/q/src/config"
-	"git.akyoto.dev/cli/q/src/os/common"
+	"git.akyoto.dev/cli/q/src/exe"
 	"git.akyoto.dev/cli/q/src/sizeof"
 )
 
@@ -339,7 +339,7 @@ restart:
 	data, dataLabels = a.Data.Finalize()
 
 	dataStart := Address(config.BaseAddress) + config.CodeOffset + Address(len(code))
-	dataStart += int32(common.Padding(dataStart, config.Align))
+	dataStart += exe.Padding(dataStart, config.Align)
 
 	for _, pointer := range dataPointers {
 		address := dataStart + pointer.Resolve()

src/compiler/Result.go

@@ -10,11 +10,11 @@ import (
 	"git.akyoto.dev/cli/q/src/asm"
 	"git.akyoto.dev/cli/q/src/config"
 	"git.akyoto.dev/cli/q/src/core"
+	"git.akyoto.dev/cli/q/src/exe/elf"
+	"git.akyoto.dev/cli/q/src/exe/macho"
+	"git.akyoto.dev/cli/q/src/exe/pe"
 	"git.akyoto.dev/cli/q/src/os/linux"
-	"git.akyoto.dev/cli/q/src/os/linux/elf"
 	"git.akyoto.dev/cli/q/src/os/mac"
-	"git.akyoto.dev/cli/q/src/os/mac/macho"
-	"git.akyoto.dev/cli/q/src/os/windows/pe"
 )
 
 // Result contains all the compiled functions in a build.
@@ -147,14 +147,11 @@ func write(writer io.Writer, code []byte, data []byte) error {
 
 	switch config.TargetOS {
 	case "linux":
-		exe := elf.New(code, data)
+		elf.Write(buffer, code, data)
-		exe.Write(buffer)
 	case "mac":
-		exe := macho.New(code, data)
+		macho.Write(buffer, code, data)
-		exe.Write(buffer)
 	case "windows":
-		exe := pe.New(code, data)
+		pe.Write(buffer, code, data)
-		exe.Write(buffer)
 	default:
 		return fmt.Errorf("unsupported platform '%s'", config.TargetOS)
 	}

src/exe/Padding.go

@@ -1,4 +1,4 @@
-package common
+package exe
 
 // Padding calculates the padding needed to align `n` bytes with the specified alignment.
 func Padding[T int | uint | int64 | uint64 | int32 | uint32](n T, align T) T {

src/exe/elf/ELF.go

@@ -6,7 +6,7 @@ import (
 	"io"
 
 	"git.akyoto.dev/cli/q/src/config"
-	"git.akyoto.dev/cli/q/src/os/common"
+	"git.akyoto.dev/cli/q/src/exe"
 )
 
 // ELF represents an ELF file.
@@ -14,20 +14,20 @@ type ELF struct {
 	Header
 	CodeHeader ProgramHeader
 	DataHeader ProgramHeader
-	CodePadding []byte
-	Code        []byte
-	DataPadding []byte
-	Data        []byte
 }
 
-// New creates a new ELF binary.
+// Write writes the ELF64 format to the given writer.
-func New(code []byte, data []byte) *ELF {
+func Write(writer io.Writer, code []byte, data []byte) {
-	codePadding := common.Padding(HeaderSize+ProgramHeaderSize*2, config.Align)
+	const HeaderEnd = HeaderSize + ProgramHeaderSize*2
-	dataOffset := config.CodeOffset + int64(len(code))
-	dataPadding := common.Padding(dataOffset, config.Align)
-	dataOffset += dataPadding
 
-	return &ELF{
+	var (
+		codePadding = exe.Padding(HeaderEnd, config.Align)
+		codeEnd     = config.CodeOffset + len(code)
+		dataPadding = exe.Padding(codeEnd, config.Align)
+		dataStart   = codeEnd + dataPadding
+	)
+
+	elf := &ELF{
 		Header: Header{
 			Magic:                       [4]byte{0x7F, 'E', 'L', 'F'},
 			Class:                       2,
@@ -62,30 +62,23 @@ func New(code []byte, data []byte) *ELF {
 		DataHeader: ProgramHeader{
 			Type:            ProgramTypeLOAD,
 			Flags:           ProgramFlagsReadable,
-			Offset:          dataOffset,
+			Offset:          int64(dataStart),
-			VirtualAddress:  config.BaseAddress + dataOffset,
+			VirtualAddress:  int64(config.BaseAddress + dataStart),
-			PhysicalAddress: config.BaseAddress + dataOffset,
+			PhysicalAddress: int64(config.BaseAddress + dataStart),
 			SizeInFile:      int64(len(data)),
 			SizeInMemory:    int64(len(data)),
 			Align:           config.Align,
 		},
-		CodePadding: bytes.Repeat([]byte{0}, int(codePadding)),
-		Code:        code,
-		DataPadding: bytes.Repeat([]byte{0}, int(dataPadding)),
-		Data:        data,
-	}
 	}
 
-// Write writes the ELF64 format to the given writer.
-func (elf *ELF) Write(writer io.Writer) {
 	binary.Write(writer, binary.LittleEndian, &elf.Header)
 	binary.Write(writer, binary.LittleEndian, &elf.CodeHeader)
 	binary.Write(writer, binary.LittleEndian, &elf.DataHeader)
-	writer.Write(elf.CodePadding)
+	writer.Write(bytes.Repeat([]byte{0}, codePadding))
-	writer.Write(elf.Code)
+	writer.Write(code)
 
-	if len(elf.Data) > 0 {
+	if len(data) > 0 {
-		writer.Write(elf.DataPadding)
+		writer.Write(bytes.Repeat([]byte{0}, dataPadding))
-		writer.Write(elf.Data)
+		writer.Write(data)
 	}
 }

src/exe/elf/ELF_test.go

@@ -0,0 +1,12 @@
+package elf_test
+
+import (
+	"io"
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/exe/elf"
+)
+
+func TestWrite(t *testing.T) {
+	elf.Write(io.Discard, nil, nil)
+}

src/exe/elf/elf.md

@@ -4,8 +4,8 @@
 
 1. ELF header (0x00 - 0x40)
 2. Program header (0x40 - 0x78)
-3. Padding (0x78 - 0x80)
+3. Padding
-4. Machine code (0x80)
+4. Machine code
 
 ## Entry point
 
@@ -28,3 +28,11 @@ https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/binfm
 
 ELF register definitions:
 https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/include/asm/elf.h
+
+## Links
+
+- https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/elf.h
+- https://lwn.net/Articles/631631/
+- https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
+- https://www.muppetlabs.com/~breadbox/software/tiny/teensy.html
+- https://nathanotterness.com/2021/10/tiny_elf_modernized.html

src/exe/macho/MachO.go

@@ -0,0 +1,126 @@
+package macho
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"git.akyoto.dev/cli/q/src/config"
+	"git.akyoto.dev/cli/q/src/exe"
+)
+
+// MachO is the executable format used on MacOS.
+type MachO struct {
+	Header
+	PageZero   Segment64
+	CodeHeader Segment64
+	DataHeader Segment64
+	UnixThread Thread
+}
+
+// Write writes the Mach-O format to the given writer.
+func Write(writer io.Writer, code []byte, data []byte) {
+	const (
+		SizeCommands = Segment64Size*3 + ThreadSize
+		HeaderEnd    = HeaderSize + SizeCommands
+	)
+
+	var (
+		codePadding = exe.Padding(HeaderEnd, config.Align)
+		codeEnd     = config.CodeOffset + len(code)
+		dataPadding = exe.Padding(codeEnd, config.Align)
+		dataStart   = codeEnd + dataPadding
+	)
+
+	m := &MachO{
+		Header: Header{
+			Magic:             0xFEEDFACF,
+			Architecture:      CPU_X86_64,
+			MicroArchitecture: 3 | 0x80000000,
+			Type:              TypeExecute,
+			NumCommands:       4,
+			SizeCommands:      SizeCommands,
+			Flags:             FlagNoUndefs | FlagNoHeapExecution,
+			Reserved:          0,
+		},
+		PageZero: Segment64{
+			LoadCommand:  LcSegment64,
+			Length:       72,
+			Name:         [16]byte{'_', '_', 'P', 'A', 'G', 'E', 'Z', 'E', 'R', 'O'},
+			Address:      0,
+			SizeInMemory: config.BaseAddress,
+			Offset:       0,
+			SizeInFile:   0,
+			NumSections:  0,
+			Flag:         0,
+			MaxProt:      0,
+			InitProt:     0,
+		},
+		CodeHeader: Segment64{
+			LoadCommand:  LcSegment64,
+			Length:       Segment64Size,
+			Name:         [16]byte{'_', '_', 'T', 'E', 'X', 'T'},
+			Address:      config.BaseAddress,
+			SizeInMemory: uint64(codeEnd),
+			Offset:       0,
+			SizeInFile:   uint64(codeEnd),
+			NumSections:  0,
+			Flag:         0,
+			MaxProt:      ProtReadable | ProtExecutable,
+			InitProt:     ProtReadable | ProtExecutable,
+		},
+		DataHeader: Segment64{
+			LoadCommand:  LcSegment64,
+			Length:       Segment64Size,
+			Name:         [16]byte{'_', '_', 'D', 'A', 'T', 'A'},
+			Address:      uint64(config.BaseAddress + dataStart),
+			SizeInMemory: uint64(len(data)),
+			Offset:       uint64(dataStart),
+			SizeInFile:   uint64(len(data)),
+			NumSections:  0,
+			Flag:         0,
+			MaxProt:      ProtReadable,
+			InitProt:     ProtReadable,
+		},
+		UnixThread: Thread{
+			LoadCommand: LcUnixthread,
+			Len:         ThreadSize,
+			Type:        0x4,
+			Data: [43]uint32{
+				42,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				config.BaseAddress + config.CodeOffset, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+				0, 0,
+			},
+		},
+	}
+
+	binary.Write(writer, binary.LittleEndian, &m.Header)
+	binary.Write(writer, binary.LittleEndian, &m.PageZero)
+	binary.Write(writer, binary.LittleEndian, &m.CodeHeader)
+	binary.Write(writer, binary.LittleEndian, &m.DataHeader)
+	binary.Write(writer, binary.LittleEndian, &m.UnixThread)
+
+	writer.Write(bytes.Repeat([]byte{0}, int(codePadding)))
+	writer.Write(code)
+	writer.Write(bytes.Repeat([]byte{0}, int(dataPadding)))
+	writer.Write(data)
+}

src/exe/macho/macho.md

@@ -0,0 +1,16 @@
+# Mach-O
+
+## Notes
+
+MacOS is the only operating system that requires loading the headers
+explicitly with both readable and executable permissions.
+
+## Loader
+
+https://github.com/apple-oss-distributions/xnu/blob/main/EXTERNAL_HEADERS/mach-o/loader.h
+
+## Links
+
+- https://en.wikipedia.org/wiki/Mach-O
+- https://github.com/aidansteele/osx-abi-macho-file-format-reference
+- https://stackoverflow.com/questions/39863112/what-is-required-for-a-mach-o-executable-to-load

src/exe/pe/EXE.go

@@ -6,7 +6,7 @@ import (
 	"io"
 
 	"git.akyoto.dev/cli/q/src/config"
-	"git.akyoto.dev/cli/q/src/os/common"
+	"git.akyoto.dev/cli/q/src/exe"
 )
 
 const NumSections = 2
@@ -17,29 +17,28 @@ type EXE struct {
 	NTHeader
 	OptionalHeader64
 	Sections [NumSections]SectionHeader
-	CodePadding []byte
-	Code        []byte
-	DataPadding []byte
-	Data        []byte
 }
 
-// New creates a new EXE file.
+// Write writes the EXE file to the given writer.
-func New(code []byte, data []byte) *EXE {
+func Write(writer io.Writer, code []byte, data []byte) {
-	codeStart := uint32(DOSHeaderSize + NTHeaderSize + OptionalHeader64Size + SectionHeaderSize*NumSections)
+	const (
-	codePadding := common.Padding(codeStart, config.Align)
+		HeaderEnd = DOSHeaderSize + NTHeaderSize + OptionalHeader64Size + SectionHeaderSize*NumSections
-	codeStart += codePadding
+	)
 
-	dataStart := codeStart + uint32(len(code))
+	var (
-	dataPadding := common.Padding(dataStart, config.Align)
+		codePadding = exe.Padding(HeaderEnd, config.Align)
-	dataStart += dataPadding
+		codeEnd     = config.CodeOffset + len(code)
+		dataPadding = exe.Padding(codeEnd, config.Align)
+		dataStart   = codeEnd + dataPadding
+	)
 
-	imageSize := uint32(dataStart + uint32(len(data)))
+	imageSize := dataStart + len(data)
-	imageSize += common.Padding(imageSize, config.Align)
+	imageSize += exe.Padding(imageSize, config.Align)
 
-	return &EXE{
+	pe := &EXE{
 		DOSHeader: DOSHeader{
 			Magic:          [4]byte{'M', 'Z', 0, 0},
-			NTHeaderOffset: 0x40,
+			NTHeaderOffset: DOSHeaderSize,
 		},
 		NTHeader: NTHeader{
 			Signature:            [4]byte{'P', 'E', 0, 0},
@@ -53,15 +52,15 @@ func New(code []byte, data []byte) *EXE {
 			MajorLinkerVersion:          0x0E,
 			MinorLinkerVersion:          0x16,
 			SizeOfCode:                  uint32(len(code)),
-			AddressOfEntryPoint:         codeStart,
+			AddressOfEntryPoint:         config.CodeOffset,
-			BaseOfCode:                  codeStart,
+			BaseOfCode:                  config.CodeOffset,
 			ImageBase:                   config.BaseAddress,
 			SectionAlignment:            config.Align, // power of 2, must be greater than or equal to FileAlignment
 			FileAlignment:               config.Align, // power of 2
 			MajorOperatingSystemVersion: 0x06,
 			MajorSubsystemVersion:       0x06,
-			SizeOfImage:                 imageSize,
+			SizeOfImage:                 uint32(imageSize),
-			SizeOfHeaders:               codeStart, // section bodies begin here
+			SizeOfHeaders:               config.CodeOffset, // section bodies begin here
 			Subsystem:                   IMAGE_SUBSYSTEM_WINDOWS_GUI,
 			DllCharacteristics:          IMAGE_DLLCHARACTERISTICS_HIGH_ENTROPY_VA | IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE | IMAGE_DLLCHARACTERISTICS_NX_COMPAT | IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE,
 			SizeOfStackReserve:          0x100000,
@@ -92,35 +91,29 @@ func New(code []byte, data []byte) *EXE {
 			{
 				Name:            [8]byte{'.', 't', 'e', 'x', 't'},
 				VirtualSize:     uint32(len(code)),
-				VirtualAddress:  codeStart,
+				VirtualAddress:  config.CodeOffset,
 				RawSize:         uint32(len(code)), // must be a multiple of FileAlignment
-				RawAddress:      codeStart,         // must be a multiple of FileAlignment
+				RawAddress:      config.CodeOffset, // must be a multiple of FileAlignment
 				Characteristics: IMAGE_SCN_CNT_CODE | IMAGE_SCN_MEM_EXECUTE | IMAGE_SCN_MEM_READ,
 			},
 			{
 				Name:            [8]byte{'.', 'r', 'd', 'a', 't', 'a'},
 				VirtualSize:     uint32(len(data)),
-				VirtualAddress:  dataStart,
+				VirtualAddress:  uint32(dataStart),
 				RawSize:         uint32(len(data)), // must be a multiple of FileAlignment
-				RawAddress:      dataStart,         // must be a multiple of FileAlignment
+				RawAddress:      uint32(dataStart), // must be a multiple of FileAlignment
 				Characteristics: IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_MEM_READ,
 			},
 		},
-		CodePadding: bytes.Repeat([]byte{0}, int(codePadding)),
-		Code:        code,
-		DataPadding: bytes.Repeat([]byte{0}, int(dataPadding)),
-		Data:        data,
-	}
 	}
 
-// Write writes the EXE file to the given writer.
-func (pe *EXE) Write(writer io.Writer) {
 	binary.Write(writer, binary.LittleEndian, &pe.DOSHeader)
 	binary.Write(writer, binary.LittleEndian, &pe.NTHeader)
 	binary.Write(writer, binary.LittleEndian, &pe.OptionalHeader64)
 	binary.Write(writer, binary.LittleEndian, &pe.Sections)
-	binary.Write(writer, binary.LittleEndian, &pe.CodePadding)
+
-	binary.Write(writer, binary.LittleEndian, &pe.Code)
+	writer.Write(bytes.Repeat([]byte{0}, int(codePadding)))
-	binary.Write(writer, binary.LittleEndian, &pe.DataPadding)
+	writer.Write(code)
-	binary.Write(writer, binary.LittleEndian, &pe.Data)
+	writer.Write(bytes.Repeat([]byte{0}, int(dataPadding)))
+	writer.Write(data)
 }

src/exe/pe/pe.md

@@ -0,0 +1,11 @@
+## Portable Executable
+
+## Links
+
+- https://learn.microsoft.com/en-us/previous-versions/ms809762(v=msdn.10)
+- https://learn.microsoft.com/en-us/archive/msdn-magazine/2002/february/inside-windows-win32-portable-executable-file-format-in-detail
+- https://learn.microsoft.com/en-us/archive/msdn-magazine/2002/march/inside-windows-an-in-depth-look-into-the-win32-portable-executable-file-format-part-2
+- https://blog.kowalczyk.info/articles/pefileformat.html
+- https://keyj.emphy.de/win32-pe/
+- https://corkamiwiki.github.io/PE
+- https://github.com/ayaka14732/TinyPE-on-Win10

src/os/linux/Syscall.go

@@ -2,9 +2,6 @@ package linux
 
 // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/entry/syscalls/syscall_64.tbl
 const (
-	Read  = 0
 	Write = 1
-	Open  = 2
-	Close = 3
 	Exit  = 60
 )

src/os/linux/elf/ELF_test.go

@@ -1,13 +0,0 @@
-package elf_test
-
-import (
-	"io"
-	"testing"
-
-	"git.akyoto.dev/cli/q/src/os/linux/elf"
-)
-
-func TestELF(t *testing.T) {
-	exe := elf.New(nil, nil)
-	exe.Write(io.Discard)
-}

src/os/mac/macho/MachO.go

@@ -1,122 +0,0 @@
-package macho
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-
-	"git.akyoto.dev/cli/q/src/config"
-	"git.akyoto.dev/cli/q/src/os/common"
-)
-
-// MachO is the executable format used on MacOS.
-type MachO struct {
-	Header
-	Code []byte
-	Data []byte
-}
-
-// New creates a new Mach-O binary.
-func New(code []byte, data []byte) *MachO {
-	return &MachO{
-		Header: Header{
-			Magic:             0xFEEDFACF,
-			Architecture:      CPU_X86_64,
-			MicroArchitecture: 3 | 0x80000000,
-			Type:              TypeExecute,
-			NumCommands:       4,
-			SizeCommands:      Segment64Size*3 + ThreadSize,
-			Flags:             FlagNoUndefs | FlagNoHeapExecution,
-			Reserved:          0,
-		},
-		Code: code,
-		Data: data,
-	}
-}
-
-// Write writes the Mach-O format to the given writer.
-func (m *MachO) Write(writer io.Writer) {
-	binary.Write(writer, binary.LittleEndian, &m.Header)
-
-	binary.Write(writer, binary.LittleEndian, &Segment64{
-		LoadCommand:  LcSegment64,
-		Length:       72,
-		Name:         [16]byte{'_', '_', 'P', 'A', 'G', 'E', 'Z', 'E', 'R', 'O'},
-		Address:      0,
-		SizeInMemory: config.BaseAddress,
-		Offset:       0,
-		SizeInFile:   0,
-		NumSections:  0,
-		Flag:         0,
-		MaxProt:      0,
-		InitProt:     0,
-	})
-
-	codePadding := common.Padding(HeaderSize+m.Header.SizeCommands, config.Align)
-	codeEnd := uint64(config.CodeOffset + len(m.Code))
-	dataPadding := common.Padding(codeEnd, config.Align)
-	dataStart := codeEnd + dataPadding
-
-	binary.Write(writer, binary.LittleEndian, &Segment64{
-		LoadCommand:  LcSegment64,
-		Length:       72,
-		Name:         [16]byte{'_', '_', 'T', 'E', 'X', 'T'},
-		Address:      config.BaseAddress,
-		SizeInMemory: config.CodeOffset + uint64(len(m.Code)),
-		Offset:       0,
-		SizeInFile:   config.CodeOffset + uint64(len(m.Code)),
-		NumSections:  0,
-		Flag:         0,
-		MaxProt:      ProtReadable | ProtExecutable,
-		InitProt:     ProtReadable | ProtExecutable,
-	})
-
-	binary.Write(writer, binary.LittleEndian, &Segment64{
-		LoadCommand:  LcSegment64,
-		Length:       72,
-		Name:         [16]byte{'_', '_', 'D', 'A', 'T', 'A'},
-		Address:      config.BaseAddress + dataStart,
-		SizeInMemory: uint64(len(m.Data)),
-		Offset:       dataStart,
-		SizeInFile:   uint64(len(m.Data)),
-		NumSections:  0,
-		Flag:         0,
-		MaxProt:      ProtReadable,
-		InitProt:     ProtReadable,
-	})
-
-	binary.Write(writer, binary.LittleEndian, &Thread{
-		LoadCommand: LcUnixthread,
-		Len:         184,
-		Type:        0x4,
-		Data: [43]uint32{
-			42,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			config.BaseAddress + config.CodeOffset, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-			0, 0,
-		},
-	})
-
-	writer.Write(bytes.Repeat([]byte{0}, int(codePadding)))
-	writer.Write(m.Code)
-	writer.Write(bytes.Repeat([]byte{0}, int(dataPadding)))
-	writer.Write(m.Data)
-}