vgabios: Initial support for fixing up assembler to workaround x86emu.

Perform post-processing of the vgabios assembler to remove certain
instructions that gcc generates and x86emu can't handle.

Signed-off-by: Kevin O'Connor <kevin@koconnor.net>

Makefile

@@ -60,6 +60,7 @@ export HOSTCC             := $(CC)
 export CONFIG_SHELL       := sh
 export KCONFIG_AUTOHEADER := autoconf.h
 export KCONFIG_CONFIG     := $(CURDIR)/.config
+AS=as
 OBJCOPY=objcopy
 OBJDUMP=objdump
 STRIP=strip
@@ -193,7 +194,12 @@ SRCVGA=src/output.c src/util.c src/pci.c \
 
 CFLAGS16VGA = $(CFLAGS16INC) -Isrc
 
-$(OUT)vgaccode16.o: $(OUT)autoconf.h ; $(call whole-compile, $(CFLAGS16VGA), $(SRCVGA),$@)
+$(OUT)vgaccode16.raw.s: $(OUT)autoconf.h ; $(call whole-compile, $(CFLAGS16VGA) -S, $(SRCVGA),$@)
+
+$(OUT)vgaccode16.o: $(OUT)vgaccode16.raw.s
+	@echo "  Fixup VGA rom assembler"
+	$(Q)$(PYTHON) ./tools/vgafixup.py $< $(OUT)vgaccode16.s
+	$(Q)$(AS) --32 src/code16gcc.s $(OUT)vgaccode16.s -o $@
 
 $(OUT)vgaentry.o: vgaentry.S $(OUT)autoconf.h
 	@echo "  Compiling (16bit) $@"

tools/vgafixup.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# Work around x86emu bugs by replacing problematic instructions.
+#
+# Copyright (C) 2012  Kevin O'Connor <kevin@koconnor.net>
+#
+# This file may be distributed under the terms of the GNU GPLv3 license.
+
+# The x86emu code widely used in Linux distributions when running Xorg
+# in vesamode is known to have issues with "retl", "leavel", "entryl",
+# and some variants of "calll".  This code modifies those instructions
+# (ret and leave) that are known to be generated by gcc to avoid
+# triggering the x86emu bugs.
+
+# It is also known that the Windows vgabios emulator has issues with
+# addressing negative offsets to the %esp register.  That has been
+# worked around by not using the gcc paremeter "-fomit-frame-pointer"
+# when compiling.
+
+import sys
+
+def main():
+    infilename, outfilename = sys.argv[1:]
+    infile = open(infilename, 'rb')
+    out = []
+    for line in infile:
+        sline = line.strip()
+        if sline == 'ret':
+            out.append('retw $2\n')
+        elif sline == 'leave':
+            out.append('movl %ebp, %esp ; popl %ebp\n')
+        else:
+            out.append(line)
+    infile.close()
+    outfile = open(outfilename, 'wb')
+    outfile.write(''.join(out))
+    outfile.close()
+
+if __name__ == '__main__':
+    main()