Basically, any time you use a named capturing group, it actually pushes the capture onto a named stack. You can then pop this stack by using the same capturing group prefixed with a hyphen, like (?<-stackToPop>).

Of course, anyone who finds themselves in this situation is going to ask: can it match XML?

It’s possible that I am missing something completely (it is rather late at night), but … very nearly. I haven’t quite figured out a nested section in the local DTD subset, but no one uses that feature anyway. (Can you spot it?)

Aside from that, most of the well-formedness criteria are handled (the obvious one being element nesting). Things that require non-local information such as entities aren’t handled. I think it is possible to handle duplicate attribute names in this form as well (via a lookahead for duplicate names).

Here is the code, and a test file which shows some stuff that is caught by this. Breaking any of the elements should make it fail:

var surrogate = @"([\ud800-\udbff][\udc00-\udfff])";// .NET can't handle \U10000-\u10FFFF
var c = @"([\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd]|"+surrogate + ")"; 
var s = @"([\u0020\u0009\u000d\u000a]+)";
var nameStartChar = @"([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|" + surrogate + ")";
var nameChar = "(" + nameStartChar + @"|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])";
var name = "(?'name'" + nameStartChar + nameChar + "*)";
var names = "(?'names'" + name + @"(\u0020" + name +")*)";
var nmtoken = "(?'nmtoken'" + nameChar + "+)";
var nmtokens = "(?'nmtokens'" + nmtoken + @"(\u0020" + nmtoken +")*)";
var pereference = "%" + name + ";";
var entityReference= "(?'entityRef'&" + name + ";)";
var charref = @"&\#([0-9]+|x[0-9a-fA-F]+);";
var reference = "(?'reference'"+ entityReference + "|" + charref + ")";
var entityValue = "(?'entityValue'\"([^%&\"]|" + pereference + "|" + reference + ")*\"|'([^%&']|" + pereference + "|" + reference + ")*')";
var eq = "(?'eq'" + s + "?=" + s + "?)";
var versionNum  = @"1\.[0-9]+";
var comment = "(?'comment'<!--((?!--)" + c + ")*-->)";
var PITarget = "(?'pitarget'(?![xX][mM][lL])"+name+")";
var PI= @"(?'PI'<\?" + PITarget + "(" + s + @"((?!\?>)" + c + @")*)?\?>)";
var misc = "(?'misc'" + comment + "|" + PI + "|" + s + ")";
var versionInfo = "(?'versionInfo'"+ s + "version" + eq + "('" + versionNum + "'|\"" + versionNum + "\"))";
var encName = "(?'encName'[A-Za-z][A-Za-z0-9._-]*)";
var encodingDecl = "(?'encodingDecl'" + s + "encoding" + eq + "(\"" + encName + "\"|'"+ encName + "'))";
var sddecl = "(?'sddecl'" + s + "standalone" + eq + "(\"(yes|no)\"|'(yes|no)'))";
var xmlDecl = @"(?'xmlDecl'<\?xml" + versionInfo + encodingDecl + "?" + sddecl + "?" + s + @"?\?>)"; 
var mixed = @"(?'mixed'\(" + s + @"?\#PCDATA" + "(" + s + @"?\|" + s + "?" + name +")*" + s + "?" + @"\)\*|\(" +s + @"?\#PCDATA" + s + @"?\))";
var children = @"(?'children'unsureifpossible)";
var contentSpec = "(?'contentspec'EMPTY|ANY|"+mixed+"|"+children+")";
var elementDecl = "(?'elementdecl'<!ELEMENT" + s + name + s + contentSpec + s + "?>)";
var stringType = "CDATA";
var tokenizedType = "(ID(REF(S)?)?|ENTIT(Y|IES)|NMTOKENS?)";
var notationType = "(?'notation'NOTATION" +s + @"\(" + s + "?" + name + "(" + s + @"?\|" + s + "?" + name + ")*" + s + @"?\))";
var enumeration = @"(?'enumeration'\(" + s + "?" + nmtoken + "(" + s + @"?\|" + s + "?" + nmtoken + ")*" + s + @"?\))";
var enumeratedType = "(?'enumType'" + notationType + "|" + enumeration +")";
var attType = "(?'attType'" + stringType + "|" + tokenizedType + "|" + enumeratedType + ")";
var attValue = "(?'attValue'\"([^<&\"]|" +reference+ ")*\"|'([^<&']|" + reference + ")*')";
var defaultDecl = @"(?'defaultDecl'\#REQUIRED|\#IMPLIED|(\#FIXED"+s+")?" + attValue + ")";
var attDef = "(?'attDef'"+ s + name + s + attType + s + defaultDecl + ")";
var attListDecl = "(?'attlist'<!ATTLIST" + s + name + attDef + "*" + s + "?>)";
var systemLiteral = "(?'systemLiteral'\"[^\"]*\"|'[^']*')";
var pubIdChar = @"[a-zA-Z0-9'()+,./:=?;!*#@$_%\u0020\u000d\u000a-]";
var pubidLiteral = "(?'pubIdLiteral'\"" + pubIdChar + "*\"|'((?!')" + pubIdChar + ")*')";
var externalID = "(?'externalID'SYSTEM" + s + systemLiteral +"|PUBLIC" + s + pubidLiteral + s + systemLiteral+")";
var nDataDecl = "(?'ndatadecl'"+s + "NDATA" + s + name + ")";
var entityDef  = "(?'entityDef'" + entityValue + "|(" +externalID + nDataDecl + "?))";
var peDef = "(?'pedef'" + entityValue + "|"  + externalID + ")";
var GEDecl = "(?'gedecl'<!ENTITY" + s + name + s + entityDef + s + "?>)";
var PEDecl = "(?'gedecl'<!ENTITY" + s + "%" + s + name + s + peDef + s + "?>)";
var entityDecl = "(?'entityDecl'"+ GEDecl + "|" + PEDecl +")";
var publicID = "(?'publicID'PUBLIC" + s + pubidLiteral + ")";
var notationDecl = "(?'notationDecl'<!NOTATION" +  s + name + s + "(" + externalID + "|" + publicID + ")" + s + "?>)";
var markupDecl = "(?'markupdecl'" + elementDecl + "|" + attListDecl + "|" + entityDecl + "|" + notationDecl + "|" + PI + "|" + comment + ")";
var DeclSep = "(?'declSep'" + pereference + "|" + s + ")";
var intSubSet = @"(?'intSubSet'(" + markupDecl + "|" + DeclSep + ")*)"; 
var docTypeDecl = "(?'doctypedecl'<!DOCTYPE" + s + name + "(" + s + externalID+ ")?" + s + @"?(\[" + intSubSet + @"\]" + s + "?)?>)"; 
var prolog = xmlDecl + "?" + misc + "*(" + docTypeDecl + misc + "*)?"; 
var attribute = "(?'attribute'" +name + eq + attValue + ")";
var CDSect = @"(?'CDSect'<!\[CDATA\[((?!\]\]>)"+c+@")*\]\]>)";
var charData = @"(((?!\]\]>)[^<&])*)";
var content = @"(?>" + // minor optimization... don't backtrack over this (makes failing faster)
		@"<(?'openclose'" + name + @")(" + s + attribute + ")*" + s + @"?/>|"+
		@"<(?'open'"+ name +@")(" + s + attribute + ")*" + s + @"?>|"+
		@"</(?=\k'open'" + s + @"?>)(?'close-open'" + name + ")" + s +@"?>|"
		+reference+@"|"
		+PI+@"|"
		+comment+@"|"
		+CDSect+@"|"
		+charData+@")*" + 
	"(?(open)(?!))";
var rootElement = @"(?'root'(<(?'rootName'" + name + ")(" + s + attribute + ")*" + s + @"?>" + content + @"</\k'rootName'" + s + "?>)|(<(?'rootName'" + name + ")(" + s + attribute + ")*" + s + @"?/>))";
 
var document = "^" + prolog + rootElement + misc + "*" + "$";
 
var testDoc = @"<?xml version='1.0' encoding=""utf-8""?><!DOCTYPE nothtml []><items>
	<item available=""yes"" >
		<name> laptop  </name>
		<![CDATA[something14!$]] 1412]]>
		<"+"\U00010000"+@"quantity>  2 &amp; &#121; &#x234f; </"+"\U00010000"+@"quantity>
	</item><?notxml?>" /* or <?xml?> here */ +@"
	<item available=""yes"" x='' y=""&amp;"">
		<name> mouse </name >
		<quantity> 1 " + /* or ]]> invalid here */  @" </quantity>
	</item>
	<item available=""no"" >
		<!----> <!-- --> <!-- - -->" + /* or <!-- -- --> here */ @"
		<name> keyboad </name>
		<quantity> 0</quantity>
	</item>
</items><!-- stuff can go here --> <!-- yup --> <?pi aasd as!@*&$^!*@&$!@ ?>";
 
//Console.WriteLine(document);
Console.WriteLine(Regex.Match(testDoc, document, RegexOptions.IgnorePatternWhitespace|RegexOptions.Singleline|RegexOptions.ExplicitCapture));

And here’s the regex (with apologies to Mail::RFC822::Address):

^(?'xmlDecl'<\?xml(?'versionInfo'([\u0020\u0009\u000d\u000a]+)version(?'eq'([\u0
020\u0009\u000d\u000a]+)?=([\u0020\u0009\u000d\u000a]+)?)('1\.[0-9]+'|"1\.[0-9]+
"))(?'encodingDecl'([\u0020\u0009\u000d\u000a]+)encoding(?'eq'([\u0020\u0009\u00
0d\u000a]+)?=([\u0020\u0009\u000d\u000a]+)?)("(?'encName'[A-Za-z][A-Za-z0-9._-]*
)"|'(?'encName'[A-Za-z][A-Za-z0-9._-]*)'))?(?'sddecl'([\u0020\u0009\u000d\u000a]
+)standalone(?'eq'([\u0020\u0009\u000d\u000a]+)?=([\u0020\u0009\u000d\u000a]+)?)
("(yes|no)"|'(yes|no)'))?([\u0020\u0009\u000d\u000a]+)?\?>)?(?'misc'(?'comment'<
!--((?!--)([\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\udbff][\udc0
0-\udfff])))*-->)|(?'PI'<\?(?'pitarget'(?![xX][mM][lL])(?'name'([:A-Z_a-z\u00C0-
\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u
218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc0
0-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F
-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\
uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040]
)*))(([\u0020\u0009\u000d\u000a]+)((?!\?>)([\u0009\u000a\u000d\u0020-\ud7ff\ue00
0-\ufffd]|([\ud800-\udbff][\udc00-\udfff])))*)?\?>)|([\u0020\u0009\u000d\u000a]+
))*((?'doctypedecl'<!DOCTYPE([\u0020\u0009\u000d\u000a]+)(?'name'([:A-Z_a-z\u00C
0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-
\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\ud
c00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u03
7F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0
-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u204
0])*)(([\u0020\u0009\u000d\u000a]+)(?'externalID'SYSTEM([\u0020\u0009\u000d\u000
a]+)(?'systemLiteral'"[^"]*"|'[^']*')|PUBLIC([\u0020\u0009\u000d\u000a]+)(?'pubI
dLiteral'"[a-zA-Z0-9'()+,./:=?;!*#@$_%\u0020\u000d\u000a-]*"|'((?!')[a-zA-Z0-9'(
)+,./:=?;!*#@$_%\u0020\u000d\u000a-])*')([\u0020\u0009\u000d\u000a]+)(?'systemLi
teral'"[^"]*"|'[^']*')))?([\u0020\u0009\u000d\u000a]+)?(\[(?'intSubSet'((?'marku
pdecl'(?'elementdecl'<!ELEMENT([\u0020\u0009\u000d\u000a]+)(?'name'([:A-Z_a-z\u0
0C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u207
0-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\
udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u
037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFD
F0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2
040])*)([\u0020\u0009\u000d\u000a]+)(?'contentspec'EMPTY|ANY|(?'mixed'\(([\u0020
\u0009\u000d\u000a]+)?\#PCDATA(([\u0020\u0009\u000d\u000a]+)?\|([\u0020\u0009\u0
00d\u000a]+)?(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u0
37D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDC
F\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-
\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u
2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[
-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*))*([\u0020\u0009\u000d\u000a]+)?\)\*|\(
([\u0020\u0009\u000d\u000a]+)?\#PCDATA([\u0020\u0009\u000d\u000a]+)?\))|(?'child
ren'unsureifpossible))([\u0020\u0009\u000d\u000a]+)?>)|(?'attlist'<!ATTLIST([\u0
020\u0009\u000d\u000a]+)(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02F
F\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\
uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u
00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u21
8F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-
\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*)(?'attDef'([\u0020\u0009\u00
0d\u000a]+)(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037
D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\
uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u
00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2F
EF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.
0-9\u00B7\u0300-\u036F\u203F-\u2040])*)([\u0020\u0009\u000d\u000a]+)(?'attType'C
DATA|(ID(REF(S)?)?|ENTIT(Y|IES)|NMTOKENS?)|(?'enumType'(?'notation'NOTATION([\u0
020\u0009\u000d\u000a]+)\(([\u0020\u0009\u000d\u000a]+)?(?'name'([:A-Z_a-z\u00C0
-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\
u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc
00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037
F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-
\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040
])*)(([\u0020\u0009\u000d\u000a]+)?\|([\u0020\u0009\u000d\u000a]+)?(?'name'([:A-
Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u2
00D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\
udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-
\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\u
FDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u
203F-\u2040])*))*([\u0020\u0009\u000d\u000a]+)?\))|(?'enumeration'\(([\u0020\u00
09\u000d\u000a]+)?(?'nmtoken'(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\
u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF
900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u
036F\u203F-\u2040])+)(([\u0020\u0009\u000d\u000a]+)?\|([\u0020\u0009\u000d\u000a
]+)?(?'nmtoken'(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u
037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFD
F0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2
040])+))*([\u0020\u0009\u000d\u000a]+)?\))))([\u0020\u0009\u000d\u000a]+)(?'defa
ultDecl'\#REQUIRED|\#IMPLIED|(\#FIXED([\u0020\u0009\u000d\u000a]+))?(?'attValue'
"([^<&"]|(?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6
\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u
3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_
a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200
D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\ud
bff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())|&\#([0-9]+|
x[0-9a-fA-F]+);()))*"|'([^<&']|(?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\u00
C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070
-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\u
dc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u0
37F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF
0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u20
40])*);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*')))*([\u0020\u0009\u000d\u000a]+)?>)|
(?'entityDecl'(?'gedecl'<!ENTITY([\u0020\u0009\u000d\u000a]+)(?'name'([:A-Z_a-z\
u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2
070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff]
[\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D
\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\u
FDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\
u2040])*)([\u0020\u0009\u000d\u000a]+)(?'entityDef'(?'entityValue'"([^%&"]|%(?'n
ame'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\
u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|(
[\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02
FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF
\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300
-\u036F\u203F-\u2040])*);|(?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u
00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u21
8F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-
\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\
u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uF
FFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*
);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*"|'([^%&']|%(?'name'([:A-Z_a-z\u00C0-\u00D6
\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u
2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udf
ff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FF
F\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]
|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);|(
?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u0
2FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7F
F\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-
\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u
218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc0
0-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-
F]+);()))*')|((?'externalID'SYSTEM([\u0020\u0009\u000d\u000a]+)(?'systemLiteral'
"[^"]*"|'[^']*')|PUBLIC([\u0020\u0009\u000d\u000a]+)(?'pubIdLiteral'"[a-zA-Z0-9'
()+,./:=?;!*#@$_%\u0020\u000d\u000a-]*"|'((?!')[a-zA-Z0-9'()+,./:=?;!*#@$_%\u002
0\u000d\u000a-])*')([\u0020\u0009\u000d\u000a]+)(?'systemLiteral'"[^"]*"|'[^']*'
))(?'ndatadecl'([\u0020\u0009\u000d\u000a]+)NDATA([\u0020\u0009\u000d\u000a]+)(?
'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FF
F\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]
|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u
02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7
FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u03
00-\u036F\u203F-\u2040])*))?))([\u0020\u0009\u000d\u000a]+)?>)|(?'gedecl'<!ENTIT
Y([\u0020\u0009\u000d\u000a]+)%([\u0020\u0009\u000d\u000a]+)(?'name'([:A-Z_a-z\u
00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u20
70-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][
\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\
u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uF
DF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u
2040])*)([\u0020\u0009\u000d\u000a]+)(?'pedef'(?'entityValue'"([^%&"]|%(?'name'(
[:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C
-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud8
00-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0
370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF90
0-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u03
6F\u203F-\u2040])*);|(?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\
u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2
C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udff
f]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF
\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|
([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())
|&\#([0-9]+|x[0-9a-fA-F]+);()))*"|'([^%&']|%(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D
8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-
\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))
(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u20
0C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\u
d800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);|(?'ref
erence'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u
0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF9
00-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D
6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\
u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\ud
fff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);
()))*')|(?'externalID'SYSTEM([\u0020\u0009\u000d\u000a]+)(?'systemLiteral'"[^"]*
"|'[^']*')|PUBLIC([\u0020\u0009\u000d\u000a]+)(?'pubIdLiteral'"[a-zA-Z0-9'()+,./
:=?;!*#@$_%\u0020\u000d\u000a-]*"|'((?!')[a-zA-Z0-9'()+,./:=?;!*#@$_%\u0020\u000
d\u000a-])*')([\u0020\u0009\u000d\u000a]+)(?'systemLiteral'"[^"]*"|'[^']*')))([\
u0020\u0009\u000d\u000a]+)?>))|(?'notationDecl'<!NOTATION([\u0020\u0009\u000d\u0
00a]+)(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u03
7F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0
-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\
u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3
001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u
00B7\u0300-\u036F\u203F-\u2040])*)([\u0020\u0009\u000d\u000a]+)((?'externalID'SY
STEM([\u0020\u0009\u000d\u000a]+)(?'systemLiteral'"[^"]*"|'[^']*')|PUBLIC([\u002
0\u0009\u000d\u000a]+)(?'pubIdLiteral'"[a-zA-Z0-9'()+,./:=?;!*#@$_%\u0020\u000d\
u000a-]*"|'((?!')[a-zA-Z0-9'()+,./:=?;!*#@$_%\u0020\u000d\u000a-])*')([\u0020\u0
009\u000d\u000a]+)(?'systemLiteral'"[^"]*"|'[^']*'))|(?'publicID'PUBLIC([\u0020\
u0009\u000d\u000a]+)(?'pubIdLiteral'"[a-zA-Z0-9'()+,./:=?;!*#@$_%\u0020\u000d\u0
00a-]*"|'((?!')[a-zA-Z0-9'()+,./:=?;!*#@$_%\u0020\u000d\u000a-])*')))([\u0020\u0
009\u000d\u000a]+)?>)|(?'PI'<\?(?'pitarget'(?![xX][mM][lL])(?'name'([:A-Z_a-z\u0
0C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u207
0-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\
udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u
037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFD
F0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2
040])*))(([\u0020\u0009\u000d\u000a]+)((?!\?>)([\u0009\u000a\u000d\u0020-\ud7ff\
ue000-\ufffd]|([\ud800-\udbff][\udc00-\udfff])))*)?\?>)|(?'comment'<!--((?!--)([
\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\udbff][\udc00-\udfff])))
*-->))|(?'declSep'%(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u03
70-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900
-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\
u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2
C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udff
f]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);|([\u0020\u0009\u000d\u000a]+)))
*)\]([\u0020\u0009\u000d\u000a]+)?)?>)(?'misc'(?'comment'<!--((?!--)([\u0009\u00
0a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\udbff][\udc00-\udfff])))*-->)|(?'P
I'<\?(?'pitarget'(?![xX][mM][lL])(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u0
0F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u300
1-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z
\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u
2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff
][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*))(([\u0020\u0009\u0
00d\u000a]+)((?!\?>)([\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\ud
bff][\udc00-\udfff])))*)?\?>)|([\u0020\u0009\u000d\u000a]+))*)?(?'root'(<(?'root
Name'(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037
F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-
\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u
00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u30
01-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u0
0B7\u0300-\u036F\u203F-\u2040])*))(([\u0020\u0009\u000d\u000a]+)(?'attribute'(?'
name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF
\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|
([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u0
2FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7F
F\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u030
0-\u036F\u203F-\u2040])*)(?'eq'([\u0020\u0009\u000d\u000a]+)?=([\u0020\u0009\u00
0d\u000a]+)?)(?'attValue'"([^<&"]|(?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\
u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2
070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff]
[\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D
\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\u
FDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\
u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*"|'([^<&']|(?'reference'(?'entityRe
f'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-
\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\u
FFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00
F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001
-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B
7\u0300-\u036F\u203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*')))*([\u0020
\u0009\u000d\u000a]+)?>(?><(?'openclose'(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u
00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2F
EF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:
A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\
u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800
-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*))(([\u0020\u
0009\u000d\u000a]+)(?'attribute'(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00
F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001
-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\
u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2
070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff]
[\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*)(?'eq'([\u0020\u0009
\u000d\u000a]+)?=([\u0020\u0009\u000d\u000a]+)?)(?'attValue'"([^<&"]|(?'referenc
e'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-
\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\u
FDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00
D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00
-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff])
)|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*
"|'([^<&']|(?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00
F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF
\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-
Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u2
00D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\
udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())|&\#([0-9]
+|x[0-9a-fA-F]+);()))*')))*([\u0020\u0009\u000d\u000a]+)?/>|<(?'open'(?'name'([:
A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\
u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800
-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u037
0-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-
\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F
\u203F-\u2040])*))(([\u0020\u0009\u000d\u000a]+)(?'attribute'(?'name'([:A-Z_a-z\
u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2
070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff]
[\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D
\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\u
FDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\
u2040])*)(?'eq'([\u0020\u0009\u000d\u000a]+)?=([\u0020\u0009\u000d\u000a]+)?)(?'
attValue'"([^<&"]|(?'reference'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00
D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00
-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff])
)(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u2
00C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\
ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())|&\
#([0-9]+|x[0-9a-fA-F]+);()))*"|'([^<&']|(?'reference'(?'entityRef'&(?'name'([:A-
Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u2
00D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\
udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-
\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\u
FDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u
203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*')))*([\u0020\u0009\u000d\u00
0a]+)?>|</(?=\k'open'([\u0020\u0009\u000d\u000a]+)?>)(?'close-open'(?'name'([:A-
Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u2
00D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\
udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-
\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\u
FDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u
203F-\u2040])*))([\u0020\u0009\u000d\u000a]+)?>|(?'reference'(?'entityRef'&(?'na
me'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u
200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([
\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02F
F\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\
uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-
\u036F\u203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);())|(?'PI'<\?(?'pitarget'(
?![xX][mM][lL])(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\
u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uF
DCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D
8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-
\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))
|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*))(([\u0020\u0009\u000d\u000a]+)((?!\?
>)([\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\udbff][\udc00-\udfff
])))*)?\?>)|(?'comment'<!--((?!--)([\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd
]|([\ud800-\udbff][\udc00-\udfff])))*-->)|(?'CDSect'<!\[CDATA\[((?!\]\]>)([\u000
9\u000a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\udbff][\udc00-\udfff])))*\]\]
>)|(((?!\]\]>)[^<&])*))*(?(open)(?!))</\k'rootName'([\u0020\u0009\u000d\u000a]+)
?>)|(<(?'rootName'(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u037
0-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-
\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u
00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C
00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff
]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*))(([\u0020\u0009\u000d\u000a]+)(?'
attribute'(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D
\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\u
FDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u0
0F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FE
F\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0
-9\u00B7\u0300-\u036F\u203F-\u2040])*)(?'eq'([\u0020\u0009\u000d\u000a]+)?=([\u0
020\u0009\u000d\u000a]+)?)(?'attValue'"([^<&"]|(?'reference'(?'entityRef'&(?'nam
e'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u2
00C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\
ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF
\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\u
F900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\
u036F\u203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*"|'([^<&']|(?'referenc
e'(?'entityRef'&(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-
\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\u
FDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-z\u00C0-\u00D6\u00
D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00
-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff])
)|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*);())|&\#([0-9]+|x[0-9a-fA-F]+);()))*
')))*([\u0020\u0009\u000d\u000a]+)?/>))(?'misc'(?'comment'<!--((?!--)([\u0009\u0
00a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\udbff][\udc00-\udfff])))*-->)|(?'
PI'<\?(?'pitarget'(?![xX][mM][lL])(?'name'([:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u
00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u30
01-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbff][\udc00-\udfff]))(([:A-Z_a-
z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\
u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|([\ud800-\udbf
f][\udc00-\udfff]))|[-.0-9\u00B7\u0300-\u036F\u203F-\u2040])*))(([\u0020\u0009\u
000d\u000a]+)((?!\?>)([\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd]|([\ud800-\u
dbff][\udc00-\udfff])))*)?\?>)|([\u0020\u0009\u000d\u000a]+))*$

using System;
 
namespace ConsoleApplication
{
    public static class MainClass
    {
        // Example:
 
        public static void Main(string[] args)
        {
            var stack = Stack.New<int>(); // real type Stack<Z,int>
 
            var s1 = stack.Push(1).Push(2).Push(3); // real type Stack<S<S<S<Z>>>>
            var s2 = s1.Pop().Pop().Pop(); // real type Stack<Z,int>
 
            // statically disallowed:
            // s2.Pop();
 
            Pop2(s1);
 
            Console.WriteLine(s1.IsEmpty());
            Console.WriteLine(s2.IsEmpty());
        }
 
        // Example of a method requiring "at least 2" items on stack.
        static void Pop2<TRest,T>(Stack<S<S<TRest>>,T> stack)
        {
            T x1;
            T x2;
            stack.Pop(out x1).Pop(out x2);
            Console.WriteLine("{0}, {1}", x1, x2);
        }
    }
 
    // Implementation:
 
    // very weak type level naturals
    // Z = zero, S<x> = successor of x.
    // so, S<Z> = 1, S<S<Z>> = 2, etc.
    public sealed class S<TPrev> {}
    public sealed class Z {}
 
    // internal node type for the linked list
    internal class StackNode<T>
    {
        public StackNode(T item, StackNode<T> next)
        {
            this.item = item;
            this.next = next;
        }
 
        internal T item;
        internal StackNode<T> next;
    }
 
    // the stack wrapper type (sized with types)
    public class Stack<TSize,T>
    {
        internal StackNode<T> node;
 
        // Construct wrapper:
        internal Stack(StackNode<T> node)
        {
            this.node = node;
        }
 
        // Push:
        internal Stack(T item, StackNode<T> node)
        {
            this.node = new StackNode<T>(item, node);
        }
 
        // note that the return type has the size increased by 1
        // S<TSize> ~= TSize++ 
        public Stack<S<TSize>,T> Push(T item)
        {
            return new Stack<S<TSize>,T>(item, node);
        }
    }
 
    // extension methods that perform the operations:
    public static class Stack
    {
        // a new stack has size zero
        public static Stack<Z,T> New<T>()
        {
            return new Stack<Z, T>(null);
        }
 
        // pop with ignored result
        public static Stack<TRest,T> Pop<TRest,T>(this Stack<S<TRest>,T> stack)
        {
            T ignored;
            return stack.Pop(out ignored);
        }
 
        // popping an item removes one S<> from the size (~= TSize--)
        public static Stack<TRest,T> Pop<TRest,T>(this Stack<S<TRest>,T> stack, out T item)
        {
            item = stack.node.item;
            return new Stack<TRest, T>(stack.node.next);
        }
 
        // These are statically known, because of the types:
        public static bool IsEmpty<T>(this Stack<Z,T> stack) { return true; }
        public static bool IsEmpty<TRest, T>(this Stack<S<TRest>, T> stack) { return false; }
 
    }
}

One fun thing about this is that we never have to check whether we’ve hit the bottom of the stack (node.next == null). Since the size is statically part of the type, and there’s no Pop for an empty list, we never have to check for this situation.

Preamble; choice of implementation

I decided to go with the latest and greatest version of Scheme: R⁶RS. There are currently two implementations available under Ubuntu Linux: Ikarus and Ypsilon. I installed both so I wouldn’t be swayed by any tempting extensions to the standard. Despite this, I ended up using Ikarus for most testing, as it ran quite a lot faster, although Ypsilon gave much better stack traces.

I also used the streams library for dealing with infinite lists (SRFI 41), which is included with both implementations.

Note: I’ll be providing all the code so that you should be able to copy-paste it into a new file and run it.

(import (rnrs)
    (rnrs r5rs (6)) ; provides 'delay' & 'force'
    (streams))

The Machine

Turing sought to capture the essence of computation. For this purpose he constructed an idealized machine, which can read and write symbols to an infinitely long piece of tape. We are going to model these idealized machines and see what they can do.

First, we want some types to represent the a-machines (the a is for automatic). Each machine has a set of states (m-configurations) it can be in, each of which contains a mapping from a set of symbols to a list of actions and a new m-configuration. I decided that the state would be my basic unit of construction, but twiddled the meaning of m-configuration a tiny bit so that instead of having each configuration contain a mapping from symbols, I would instead have a list of configurations, each one with a list of which symbols activate it. I ended up with the following:

; type for a configuration
(define-record-type m-cfg
            (fields symbols operations next))

This defines a new record type called m-cfg with the fields symbols, operations and next. The define-record-type form defines a constructor (make-m-cfg) and accessors for each field (m-cfg-*). Rather than have a ‘machine’ type, I decided that this would just be left implicit; if we know what the current state is then we can follow the links to next whenever we want to.

The Tape

Now, each machine operates upon an infinitely long ‘tape’. To model this, I use two streams, which are infinite lists. One is the infinite length of the tape to the left of the machine, and the other is the infinite length of the tape to the right of the machine. I decided that the first item in the right list would be the current item that the machine is reading.

; type for a tape (modeled as two stacks)
(define-record-type tape
            (fields left right min max index))

You will note that there are also the fields min, max, and index. These are solely used to track how much of the tape the machine has “visited”. Without this information, we would not know how much of the tape to show when we want to look at it; and since it is infinitely long, this could be a problem!

Representing operations

The operations that a machine can perform consist of:

• Move right
• Move left
• Print symbol
• Erase symbol
• Halt

I implemented these as an enumeration, just in case, but I didn’t actually end up utilizing any of the enumeration features.

; the operations available
(define-enumeration op
  (right left erase print halt)
  op-set)

There is a small difficulty with this representation: the ‘print’ operation needs to be able to take an argument. I decided that operations would always be passed around as lists, and that only ‘print’ would have a second element in the list: its argument. Here is a little shorthand to make this representation easier:

; these need to be lists, because print takes an argument
(define L (list (op left)))
(define R (list (op right)))
(define P (lambda (c) (list (op print) c)))
(define E (list (op erase)))
(define H (list (op halt)))

Thus, we can represent a list of operations like this: (list R R (P #\A) L E H)—that’s right, right, print ‘A’ (Scheme’s syntax for characters is a little weird), left, erase, halt.

Moving around on the tape

Here is a simple tape; it is completely empty. Note that I use the symbol 'empty to represent empty places on the tape. stream-constant makes an infinite stream of the value(s) supplied.

(define empty-tape
  (make-tape (stream-constant 'empty) (stream-constant 'empty) 0 0 0))

Next we need code to actually implement the operations described above. It is fairly straightforward, but we also have to keep track of the index and max/min points on the tape:

; Tape manipulation:
(define (current-symbol tape)
  (stream-car (tape-right tape)))
 
(define (move-item from to)
  (values (stream-cdr from)
      (stream-cons (stream-car from) to)))
 
(define (move-right tape)
  (let-values (
      [(right left) (move-item (tape-right tape) (tape-left tape))]
      [(min max) (update-min-max (tape-min tape) (tape-max tape) (+ 1 (tape-index tape)))])
    (make-tape left right min max (+ 1 (tape-index tape)))))
 
(define (move-left tape)
  (let-values (
      [(left right) (move-item (tape-left tape) (tape-right tape))]
      [(min max) (update-min-max (tape-min tape) (tape-max tape) (- (tape-index tape) 1))])
    (make-tape left right min max (- (tape-index tape) 1))))
 
(define (print tape symbol)
  (make-tape
    (tape-left tape)
    (stream-cons symbol (stream-cdr (tape-right tape)))
    (tape-min tape)
    (tape-max tape)
    (tape-index tape)))
 
(define (erase tape)
  (print tape 'empty))
 
(define (update-min-max min max i)
  (cond 
    [(< i min) (values i max)]
    [(> i max) (values min i)]
    [else      (values min max)]))

Notice that erase is actually redundant because we can just print 'empty. We also want a “dispatcher” of sorts that takes a value representing an operation and performs that operation. This is where passing the argument around with the ‘print’ came in useful:

; performs an operation on a tape, returns new tape
(define (perform-op tape oper)
  (case (car oper)
    [(op left) (move-left tape)]
    [(op right) (move-right tape)]
    [(op print) (print tape (cadr oper))]
    [(op erase) (erase tape)]
    [(op halt) #f])) ; ungraceful halt!

Running the machine

Now that we have the operations implemented, we can almost run a state against a tape. First we need to figure out just which of the configurations of the state to run. This procedure receives a list of configurations (but they are all part of the same ‘state’) and a symbol, and picks the first configuration that has a matching symbol. Note that the configurations can also have the symbol 'any, which matches anything.

; finds the correct rule to follow for this symbol
(define (find-cfg machine symbol)
  (find (lambda (cfg) (or
                (find [lambda (s) (eqv? symbol s)]
                  (m-cfg-symbols cfg))
                (find [lambda (s) (eqv? 'any s)]
                  (m-cfg-symbols cfg))))
        machine))

Now that we have a way to find out which rule to perform, and how to perform it, we can run it against a tape. This procedure advances the machine to the next state, performing all the operations needed. It returns the new state and a new tape.

; runs a machine forward one step
(define (run-machine tape machine)
  (let ([cfg (find-cfg (force machine) (current-symbol tape))])
    (list (fold-left perform-op tape (m-cfg-operations cfg))
          (m-cfg-next cfg))))

Displaying the tape

Being able to run the machine against a tape isn’t much good if we can’t see the result, so here’s a procedure to print out what it looks like. This is where we need the indexes we kept track of on the tape, so we know when to stop printing.

(define (print-tape tape)
  ; move as far left as possible
  (let ([leftTape (go-far-left tape)])
    ; now go right
    (do ([t leftTape])
      ((> (tape-index t) (tape-max tape)) t)
      (when (eqv? (tape-index tape) (tape-index t))
              (display "["))
      (cond
          [(eqv? (current-symbol t) 'empty) (display ".")]
          [else (display (current-symbol t))])
      (when (eqv? (tape-index tape) (tape-index t))
              (display "]"))
      (set! t (move-right t)))))
 
; moves to the far left of the tape (as far as has been travelled)
(define (go-far-left tape)
  (do ([t tape])
    ((= (tape-index t) (tape-min tape)) t)
     (set! t (move-left t))))

I print out a ‘.’ for each blank, and surround the current symbol with square brackets (actually I’ve changed that to be underlining in this blog post).

Turing Machines!

Now we can test it! Here is my definition of Turing’s first published machine:

; Turing's first published machine!
(define m1 (letrec 
  ([b (delay (list
       (make-m-cfg (list 'any) (list (P #\0) R) c)))]
   [c (delay (list
       (make-m-cfg (list 'any) (list R) e)))]
   [e (delay (list
       (make-m-cfg (list 'any) (list (P #\1) R) f)))]
   [f (delay (list
       (make-m-cfg (list 'any) (list R) b)))])
  b))

Now you see why we needed to import ‘delay’… since Scheme is a strictly-evaluated language, we can’t just letrec each state in terms of the others, so I wrap each one up in delay, then force it in one place; just before we use it in the run-machine procedure.

Other than that it is fairly straightforward, each state has a list of m-configurations, each of which has a list of what symbols it accepts, the actions to take, and the next state to move to. After we letrec, we have the initial state defined as the ‘machine’—in this case, b.

We can run this machine like so:

(define (go t m)
 (do ([tm (list t m)])
  ((eqv? tm #f) (car tm))
  (print-tape (car tm))
  (newline)
  (set! tm (run-machine (car tm) (cadr tm)))))
 
(go empty-tape m1)

This gives us the following output:

.
0.
0..
0.1.
0.1..
0.1.0.
0.1.0..
0.1.0.1.
0.1.0.1..
0.1.0.1.0.
...

This looks correct: P0, R, R, P1, R, R, etc.

Machine redux

The next machine Turing gives is the same as the first one, only in a different form:

; the same machine, only smaller
(define m2 (letrec
         ([b (delay (list
           (make-m-cfg (list 'empty) (list (P #\0)) b)
           (make-m-cfg (list #\0) (list R R (P #\1)) b)
           (make-m-cfg (list #\1) (list R R (P #\0)) b)))])
         b))

It has only one state, but changes depending on what the current symbol is. This produces the same output as the first machine, but in fewer steps:

.
0
0.1
0.1.0
0.1.0.1
0.1.0.1.0
0.1.0.1.0.1
0.1.0.1.0.1.0
0.1.0.1.0.1.0.1
0.1.0.1.0.1.0.1.0
...

You may be wondering why Turing leaves blanks between each printed symbol. He used the convention that only the ‘even’ squares (termed F-squares) would be the output. The ‘odd’ squares (termed E-squares) would be used as a scratch-pad.

Machine-à-trois

The third machine is a little more interesting. Whereas the first two printed out 01010101..., this prints 01011011101111011111...:

; Turing's third machine
(define m3 (letrec
         ([b (delay (list
                   (make-m-cfg (list 'any) (list (P #\ǝ) R (P #\ǝ) R (P #\0) R R (P #\0) L L) o)))]
          [o (delay (list
                  (make-m-cfg (list #\1) (list R (P #\x) L L L) o)
                  (make-m-cfg (list #\0) (list) q)))]
          [q (delay (list
                  (make-m-cfg (list #\0 #\1) (list R R) q)
                  (make-m-cfg (list 'empty) (list (P #\1) L) p)))]
          [p (delay (list
                  (make-m-cfg (list #\x) (list E R) q)
                  (make-m-cfg (list #\ǝ) (list R) f)
                  (make-m-cfg (list 'empty) (list L L) p)))]
          [f (delay (list
                  (make-m-cfg (list 'empty) (list (P #\0) L L) o)
                  (make-m-cfg (list 'any) (list R R) f)))])
         b))

With the output:

.
ǝǝ0.0
ǝǝ0.0
ǝǝ0.0
ǝǝ0.0..
ǝǝ0.0.1
ǝǝ0.0.1
ǝǝ0.0.1
ǝǝ0.0.1
ǝǝ0.0.1
ǝǝ0.0.1
ǝǝ0.0.1..
ǝǝ0.0.1.0
ǝǝ0.0.1x0
ǝǝ0.0.1x0
ǝǝ0.0.1x0
ǝǝ0.0.1x0
ǝǝ0.0.1x0..
ǝǝ0.0.1x0.1
ǝǝ0.0.1x0.1
ǝǝ0.0.1.0.1
ǝǝ0.0.1.0.1
ǝǝ0.0.1.0.1..
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1
ǝǝ0.0.1.0.1.1..
ǝǝ0.0.1.0.1.1.0
ǝǝ0.0.1.0.1.1x0
ǝǝ0.0.1.0.1x1x0
ǝǝ0.0.1.0.1x1x0
ǝǝ0.0.1.0.1x1x0
ǝǝ0.0.1.0.1x1x0
ǝǝ0.0.1.0.1x1x0
ǝǝ0.0.1.0.1x1x0..
ǝǝ0.0.1.0.1x1x0.1
ǝǝ0.0.1.0.1x1x0.1
ǝǝ0.0.1.0.1x1.0.1
ǝǝ0.0.1.0.1x1.0.1
ǝǝ0.0.1.0.1x1.0.1..
ǝǝ0.0.1.0.1x1.0.1.1
ǝǝ0.0.1.0.1x1.0.1.1
ǝǝ0.0.1.0.1x1.0.1.1
ǝǝ0.0.1.0.1x1.0.1.1
ǝǝ0.0.1.0.1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1.1..
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1..
ǝǝ0.0.1.0.1.1.0.1.1.1.0
ǝǝ0.0.1.0.1.1.0.1.1.1x0
ǝǝ0.0.1.0.1.1.0.1.1x1x0
ǝǝ0.0.1.0.1.1.0.1x1x1x0
ǝǝ0.0.1.0.1.1.0.1x1x1x0
ǝǝ0.0.1.0.1.1.0.1x1x1x0
ǝǝ0.0.1.0.1.1.0.1x1x1x0
ǝǝ0.0.1.0.1.1.0.1x1x1x0
ǝǝ0.0.1.0.1.1.0.1x1x1x0
ǝǝ0.0.1.0.1.1.0.1x1x1x0..
ǝǝ0.0.1.0.1.1.0.1x1x1x0.1
ǝǝ0.0.1.0.1.1.0.1x1x1x0.1
ǝǝ0.0.1.0.1.1.0.1x1x1.0.1
ǝǝ0.0.1.0.1.1.0.1x1x1.0.1
ǝǝ0.0.1.0.1.1.0.1x1x1.0.1..
ǝǝ0.0.1.0.1.1.0.1x1x1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1x1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1x1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1x1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1..
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1x1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1..
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1
ǝǝ0.0.1.0.1.1.0.1.1.1.0.1.1.1.1

Here we can see the scratch-pad being used. We can also see the beginnings of a pattern of execution; notice how the machine returns to the beginning of the tape after completing each set of 1s.

Opus Magnum

Next up was my main challenge. One of the major contributions of Turing’s paper was to display a machine which could emulate any other machine you wanted. In essence, you don’t actually need lots of different machines. You can just build one, and it can do anything any of the other machines can do! This is the principle behind modern general-purpose computers.

This is a very big, and complex machine. Some of the intricacies not involved in the other machines are:

• m-functions; that is, configurations which accept parameters—luckily, this was surprisingly easy to implement using Scheme; I simply wrap the delayed code inside another lambda
• m-configurations which are parametrized over all symbols on the machine—this is solved by just mapping a lambda over the list of symbols
• variadic m-functions—solved by the magic of case-lambda
• poorly-scanned journal document containing Fraktur and Greek letters

But not only was this by far the biggest and most complex machine supplied by Turing, I had read texts mentioning unspecified bugs in the program.

… and sure enough, I ran into a ‘bug’. There were some configurations used in the machine which weren’t defined in the paper! At first I thought this was due to the low resolution of the PDF, but even enhancing the image didn’t help.

After much supplication and burnt offerings to the God of the Internet, I managed to find that:

Someone else did the work already

Yay!

More specifically, I found a paper entitled Understanding Turing’s Universal Machine — Personal Style in Program Description (which is unfortunately not available to the public), a marvelous paper that not only explains the errors made in detail, but also provides a nice, corrected version of Turing’s exposition of his machine.

After painstakingly re-checking all the states again, I arrived at this:

; need this to generate a couple of cfgs
(define u-symbols (list
		   #\A #\C #\D #\0 #\1
		   #\u #\v #\w #\x #\y #\z
		   #\; #\L #\R #\N
		   #\∷ #\:
		   ))
 
; yo dawg
(define u (letrec
        (
         [f (lambda (C B a) (delay (list
                (make-m-cfg (list #\ǝ) (list L) (f1 C B a))
                (make-m-cfg (list 'any) (list L) (f C B a)))))]
         [f1 (lambda (C B a) (delay (list
                (make-m-cfg (list a) (list) C)
                (make-m-cfg (list 'empty) (list R) (f2 C B a))
                (make-m-cfg (list 'any) (list R) (f1 C B a)))))]
         [f2 (lambda (C B a) (delay (list
                (make-m-cfg (list a) (list) C)
                (make-m-cfg (list 'empty) (list R) B)
                (make-m-cfg (list 'any) (list R) (f1 C B a)))))]
         [fdash (lambda (C B a) (delay (list
                (make-m-cfg (list 'any) (list) (f (l C) B a)))))]
         [fdashdash (lambda (C B a) (delay (list
                (make-m-cfg (list 'any) (list) (f (r C) B a)))))]
         [r (lambda (C) (delay (list
                (make-m-cfg (list 'any) (list R) C))))]
         [l (lambda (C) (delay (list
                (make-m-cfg (list 'any) (list L) C))))]
         [q (case-lambda
              [(C) (delay (list
                (make-m-cfg (list 'empty) (list R) (q1 C))
                (make-m-cfg (list 'any) (list R) (q C))))]
         [(C a) (delay (list
                (make-m-cfg (list 'any) (list) (q (q1 C a)))))])]
         [q1 (case-lambda
               [(C) (delay (list
                (make-m-cfg (list 'empty) (list) C)
                (make-m-cfg (list 'any) (list R) (q C))))]
               [(C a) (delay (list
                (make-m-cfg (list a) (list) C)
                (make-m-cfg (list 'any) (list L) (q1 C a))))])]
         [pe (lambda (C b) (delay (list
                (make-m-cfg (list 'any) (list) (f (pe1 C b) c #\ǝ)))))]
         [pe1 (lambda (C b) (delay (list
                (make-m-cfg (list 'empty) (list (P b)) C)
                (make-m-cfg (list 'any) (list R R) (pe1 C b)))))]
         [pe2 (lambda (C a b) (delay (list
                (make-m-cfg (list 'any) (list) (pe (pe C b) a)))))]
         [c (lambda (C B a) (delay (list
                (make-m-cfg (list 'any) (list) (fdash (c1 C) B a)))))]
         [c1 (lambda (C) (delay (map
                    (lambda (b) (make-m-cfg (list b) (list) (pe C b)))
                      u-symbols)))]
         [ce (case-lambda
               [(C B a) (delay (list
                (make-m-cfg (list 'any) (list) (c (e C B a) B a))))]
               [(B a) (delay (list
                (make-m-cfg (list 'any) (list) (ce (ce B a) B a))))])]
         [ce2 (lambda (B a b) (delay (list
                (make-m-cfg (list 'any) (list) (ce (ce B b) a)))))]
         [ce3 (lambda (B a b g) (delay (list
                (make-m-cfg (list 'any) (list) (ce (ce2 B b g) a)))))]
         [ce5 (lambda (B a b g d e) (delay (list
                (make-m-cfg (list 'any) (list) (ce3 (ce2 B d e) a b g)))))] ; added
         [cp (lambda (C U F a b) (delay (list
                (make-m-cfg (list 'any) (list) (fdash (cp1 C U b) (f U F b) a)))))]
         [cp1 (lambda (C U b) (delay (map
                   (lambda (g) (make-m-cfg (list g) (list) (fdash (cp2 C U g) U b)))
                     u-symbols)))]
         [cp2 (lambda (C U g) (delay (list
                (make-m-cfg (list g) (list) C)
                (make-m-cfg (list 'any) (list) U))))]
         [cpe (case-lambda 
                [(C U F a b) (delay (list
                 (make-m-cfg (list 'any) (list) (cp (e (e C C b) C a) U F a b))))]
                [(U F a b) (delay (list
                 (make-m-cfg (list 'any) (list) (cpe (cpe U F a b) U F a b))))])]
         [e (case-lambda
               [(C) (delay (list
                (make-m-cfg (list #\ǝ) (list R) (e1 C))
                (make-m-cfg (list 'any) (list L) (e C))))]
               [(B a) (delay (list 
                (make-m-cfg (list 'any) (list) (e (e B a) B a))))]
               [(C B a) (delay (list 
                (make-m-cfg (list 'any) (list) (f (e1 C B a) B a))))])]
         [e1 (case-lambda
               [(C) (delay (list
                (make-m-cfg (list 'empty) (list) C) 
                (make-m-cfg (list 'any) (list R E R) (e1 C))))]
               [(C B a) (delay (list
                (make-m-cfg (list 'any) (list E) C)))])]
         [con (lambda (C a) (delay (list
                (make-m-cfg (list #\A) (list L (P a) R) (con1 C a))
                (make-m-cfg (list 'any) (list R R) (con C a)))))]
         [con1 (lambda (C a) (delay (list
                (make-m-cfg (list #\A) (list R (P a) R) (con1 C a))
                (make-m-cfg (list #\D) (list R (P a) R) (con2 C a)))))]
         [con2 (lambda (C a) (delay (list
                (make-m-cfg (list #\C) (list R (P a) R) (con2 C a))
                (make-m-cfg (list 'any) (list R R) C))))]
         [b (delay (list
                (make-m-cfg (list 'any) (list) (f b1 b1 #\∷))))]
         [b1 (delay (list
                (make-m-cfg (list 'any) (list R R (P #\:) R R (P #\D) R R (P #\A) R R (P #\D)) anf)))] ; added "R R PD"
         [anf (delay (list
                (make-m-cfg (list 'any) (list) (q anf1 #\:))))] ; corrected from "(g ..."
         [anf1 (delay (list
                (make-m-cfg (list 'any) (list) (con fom #\y))))]
         [fom (lambda () (list
                (make-m-cfg (list #\;) (list R (P #\z) L) (con fmp #\x))
                (make-m-cfg (list #\z) (list L L) fom)
                (make-m-cfg (list #\ǝ) (list H) fom)
                (make-m-cfg (list 'any) (list L) fom)))]
         [fmp (delay (list
                (make-m-cfg (list 'any) (list) (cpe (e (e anf #\x) #\y) sim #\x #\y))))] ; corrected
         [sim (delay (list
                (make-m-cfg (list 'any) (list) (fdash sim1 sim1 #\z))))]
         [sim1 (delay (list
                (make-m-cfg (list 'any) (list) (con sim2 'empty))))]
         [sim2 (delay (list
                (make-m-cfg (list #\A) (list) sim3)
                (make-m-cfg (list 'any) (list L (P #\u) R R R) sim2)))] ; corrected from "R ..."
         [sim3 (delay (list
                (make-m-cfg (list #\A) (list L (P #\y) R R R) sim3)
                (make-m-cfg (list 'any) (list L (P #\y)) (e mf #\z))))]
         [mf (delay (list
                (make-m-cfg (list 'any) (list) (q mf1 #\:))))] ; corrected from "(g mf ..."
         [mf1 (delay (list
                (make-m-cfg (list #\A) (list L L L L) mf2)
                (make-m-cfg (list 'any) (list R R) mf1)))]
         [mf2 (delay (list
                (make-m-cfg (list #\C) (list R (P #\x) L L L) mf2)
                (make-m-cfg (list #\:) (list) mf4)
                (make-m-cfg (list #\D) (list R (P #\x) L L L) mf3)))]
         [mf3 (delay (list
                (make-m-cfg (list #\:) (list) mf4)
                (make-m-cfg (list 'any) (list R (P #\v) L L L) mf3)))]
         [mf4 (delay (list
                (make-m-cfg (list 'any) (list) (con (l (l mf5)) 'empty))))]
         [mf5 (delay (list
                (make-m-cfg (list 'empty) (list (P #\:)) sh)
                (make-m-cfg (list 'any) (list R (P #\w) R) mf5)))]
         [sh (delay (list
                (make-m-cfg (list 'any) (list) (f sh1 inst #\u))))]
         [sh1 (delay (list
                (make-m-cfg (list 'any) (list L L L) sh2)))]
         [sh2 (delay (list
                (make-m-cfg (list #\D) (list R R R R) sh3) ; corrected from "sh2"
                (make-m-cfg (list 'any) (list) inst)))]
         [sh3 (delay (list
                (make-m-cfg (list #\C) (list R R) sh4)
                (make-m-cfg (list 'any) (list) inst)))]
         [sh4 (delay (list
                (make-m-cfg (list #\C) (list R R) sh5)
                (make-m-cfg (list 'any) (list) (pe2 inst #\0 #\:))))]
         [sh5 (delay (list
                (make-m-cfg (list #\C) (list) inst)
                (make-m-cfg (list 'any) (list) (pe2 inst #\1 #\:))))]
         [inst (delay (list                                 ; note that inst1 is forced here!
                                                            ; this is because it is a zero-arity varargs
                (make-m-cfg (list 'any) (list) (q (l (inst1)) #\u))))] ; corrected from "(g ..." 
         [inst1 (case-lambda
                 [() (delay (map
                       (lambda (a) (make-m-cfg (list a) (list R E) (inst1 a)))
                       u-symbols))]
                 [(x) (case x
                      [(#\L) (delay (list (make-m-cfg (list 'any) (list) (ce5 ov #\v #\y #\x #\u #\w))))]
                      [(#\R) (delay (list (make-m-cfg (list 'any) (list) (ce5 ov #\v #\x #\u #\y #\w))))]
                      [(#\N) (delay (list (make-m-cfg (list 'any) (list) (ce5 ov #\v #\x #\y #\u #\w))))])])]
         [ov (delay (list
               (make-m-cfg (list 'any) (list) (q (r (r ov1)) #\A))))] ; changed from original
         [ov1 (delay (list
               (make-m-cfg (list #\D) (list) (e anf))
               (make-m-cfg (list 'empty) (list (P #\D)) (e anf))))]
         ) b)) ;; start in state 'b'

I don’t think I can blame Turing much for the errors in the paper. It seems as though some arose through printing typos, and attempting to debug this baroque machine by hand, on paper, would have been a difficult task. (I don’t think he even had Visual Studio!)

A machine in a machine

Of course, the final test of this is to see whether this machine can actually emulate another like it is supposed to. I defined a short procedure to set up a machine on a tape according to Turing’s ingenious encoding.

(define (setup-tape tape inits) 
  (let ([t (move-right (print (move-right (print tape #\ǝ)) #\ǝ))]
  	[inits (append inits (list #\∷))])
   (go-far-left (fold-left (lambda (tape init)
  	       (move-right (move-right (print tape init)))) 
	     t inits))))

A quick explanation of Turing’s encoding

The idea is to first simplify, then encode the states. Turing noted that many of the m-configurations’ operations could be considered redundant:

• as mentioned above, instead of having “erase” we can simply print the ‘blank’ symbol
• instead of not modifying the symbol, we simply print the symbol already present

There are then only three types of operation each state needs to perform:

1. print something and go left
2. print something and go right
3. print something and stay put

States which have a sequence of operations can be split into a series of states, each of which transfers control to the next one.

Now, if we encode all the symbols and configurations as numbers, we can write them out. Turing chose to encode them via this scheme:

• configurations’ numbers are the symbol ‘D’ followed by n ‘A’s, where n is the number of the state
• symbols’ numbers are similar, with ‘D’ followed by n ‘C’s. (Turing set ‘blank’ to always be symbol 0, represented as simply “D”)

So to encode each configuration, we write down its number, the symbol it accepts, the symbol it outputs, which direction to move, and which state to go to next. We prefix each configuration with ‘;’. (When we input it into the machine we also sandwich the whole thing between ‘ǝǝ’ and ‘∷’.)

Here is the example which Turing gives in his paper. I have formatted it to make it easier to read. Can you tell what it does?

(define example (setup-tape empty-tape (list 
			     #\; #\D #\A             #\D #\D #\C     #\R #\D #\A #\A
			     #\; #\D #\A #\A         #\D #\D         #\R #\D #\A #\A #\A
			     #\; #\D #\A #\A #\A     #\D #\D #\C #\C #\R #\D #\A #\A #\A #\A
			     #\; #\D #\A #\A #\A #\A #\D #\D         #\R #\D #\A)))

If we translate the symbols to get numbers we have the following:

; 1 0 0 R 2
; 2 0 . R 3
; 3 0 1 R 4
; 4 0 . R 1

This machine prints alternately 0.1.0.1.. In fact, when I first typed it up, I left off an ‘A’ on the 3rd state and couldn’t figure out why the machine was printing 0.11111...!

Just to show it works

Here is some output of the universal machine running the ’0101′ machine. I have only included a snippet as the machine takes a while to get to this stage. You’ll also notice the output format is different from the other machines; this one outputs some state information and the output of the emulated machine (in this case, 0 and 1), separated by colons. So far, after a minute or so, the machine has output 010

.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C
.ǝǝ;.D.A.D.DuCuR.DyAyAy;.D.A.A.D.D.R.D.A.A.A.;.D.A.A.A.D.D.C.C.R.D.A.A.A.A.;.D.A.A.A.A.D.D.R.D.A.∷.:.D.A.D.:.0.:.D.C.D.A.A.D.:.D.C.D.D.A.A.A.D.:.1.:.D.C.D.D.C.C.D.A.A.A.A.D.:.D.CvDvDvCvCvDxD.A.D.:.0.:.D.C

And that’s enough for today! Feel free to post corrections, additions, your own Turing machines, and so on

If you’re wondering, the entire hierarchy is drawn from Umberto Eco‘s novel Baudolino.

If we want to restrict ourselves to assigned, non-control, non-private Unicode characters, then by my reckoning that gives us 129,775 available characters.

wget http://unicode.org/Public/UNIDATA/UnicodeData.txt
awk -F ';' UnicodeData.txt -f countUnichars.awk | bc

countUnichars.awk source:

BEGIN { print "ibase=16" } # set bc to hex mode
 
$2 ~ /Private/ { # skip any lines with "private" in the description
    getline;
}
 
n { # if n is set, then print the range for bc to calculate
    printf("(%s-%s+1)+",$1,n);
    n="";
}
 
$2 ~ /First>/ { # set n if the start of a range
    n=$1;
    getline;
}
 
$3 !~ "C." { # otherwise count anything that isn't some kind of a control character
    i++;
}
 
END { # print out the count of everything else
    printf("%X\n",i)
}

This means we can store exactly 2377 bits (297 bytes) per message (this is ), so if we use a 16-colour palette we can store about 594 pixels (), which can almost reproduce the Mona Lisa thumbnail in the contest page.

unicodedata-520d2txt

Update: Here is a version with some further edits performed in order to learn how to use Shake.

That said, the vectorization code is clearly an improvement on mine, so let’s take that ball and run with it!

The Code

Now we use SIMD instructions to vectorize the counting of characters. I modified this from Colin’s routine, and I’m sure he has some bit-fiddling up his sleeves that would make this run even faster

As it is, I used a straightforward algorithm to extract the information.

#define GetMask(x) __builtin_ia32_pmovmskb128(x)
#define LoadBytes(x) __builtin_ia32_loaddqu(x)
#define CompareEquality(x,y) __builtin_ia32_pcmpeqb128((x),(y))
#define Or(x,y) __builtin_ia32_por128((x),(y))
#define NotExpected(x) __builtin_expect((x),0)
#define And(x,y) __builtin_ia32_pand128((x),(y))
 
typedef unsigned char v16qi __attribute__ ((vector_size(16)));
 
const char mask[16] = {
    0xc0, 0xc0, 0xc0, 0xc0,
    0xc0, 0xc0, 0xc0, 0xc0,
    0xc0, 0xc0, 0xc0, 0xc0,
    0xc0, 0xc0, 0xc0, 0xc0
};
const char match[16] = {
    0x80, 0x80, 0x80, 0x80,
    0x80, 0x80, 0x80, 0x80,
    0x80, 0x80, 0x80, 0x80,
    0x80, 0x80, 0x80, 0x80
};
const char zero[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
unsigned char HammingWeight[65536]; //initialized elsewhere
 
static size_t cp_strlen_utf8_sse2(const char *_s)
{
    const char *s;
    const v16qi allZero = LoadBytes(zero);
    const v16qi masking = LoadBytes(mask);
    const v16qi matching = LoadBytes(match);
 
    v16qi row;
    size_t count = 0;
    unsigned char b;
 
    // unaligned bytes
    for (s = _s; (uintptr_t) (s) & (sizeof(v16qi) - 1); s++) {
        b = *s;
        if (b == '\0')
            goto done;
        count += (b >> 7) & ((~b) >> 6);
    }
 
    /* Handle complete blocks. */
    for (;; s += sizeof(v16qi)) {
        /* Prefetch */
        __builtin_prefetch(&s[256], 0, 0);
 
        /* Load Bytes */
        row = LoadBytes(s);
 
        /* Expect this to be false :) */
        if (NotExpected(GetMask(
                                   /* Check for zero bytes */
                                      CompareEquality(allZero, row))))
            break;
 
        /* Count number of non-starter bytes */
 
        row = And(row, masking);
        row = CompareEquality(row, matching);
        count += HammingWeight[GetMask(row)];
    }
 
    //leftover bytes
    for (;; s++) {
        b = *s;
        if (b == '\0')
            break;
        count += (b >> 7) & ((~b) >> 6);
    }
 
  done:
    return ((s - _s) - count);
}

Results

This counts about twice as fast as GCC/libc’s standard, non-UTF-8 strlen. Note the discrepancies between my timings of Colin’s code and his own tests. Damn thou, 32-bits!

"": 0 0 0 0 0 0 0
"hello, world": 12 12 12 12 12 12 12
"naïve": 6 6 6 5 5 5 5
"こんにちは": 15 15 15 5 5 5 5
"abcdefghijklmnopqrstuvwxyzβ": 28 28 28 27 27 27 27
testing 33554424 bytes of repeated "hello, world":
                      gcc_strlen =   33554424: 0.019331 +/- 0.001076
                      kjs_strlen =   33554424: 0.035095 +/- 0.000530
                       cp_strlen =   33554424: 0.021472 +/- 0.000310
                 kjs_strlen_utf8 =   33554424: 0.070260 +/- 0.000240
                  gp_strlen_utf8 =   33554424: 0.035144 +/- 0.000471
                  cp_strlen_utf8 =   33554424: 0.050539 +/- 0.000342
             cp_strlen_utf8_sse2 =   33554424: 0.010297 +/- 0.001551
testing 33554430 bytes of repeated "naïve":
                      gcc_strlen =   33554430: 0.019176 +/- 0.000824
                      kjs_strlen =   33554430: 0.035090 +/- 0.000478
                       cp_strlen =   33554430: 0.021472 +/- 0.000323
                 kjs_strlen_utf8 =   27962025: 0.070347 +/- 0.000354
                  gp_strlen_utf8 =   27962025: 0.054802 +/- 0.000299
                  cp_strlen_utf8 =   27962025: 0.050595 +/- 0.000602
             cp_strlen_utf8_sse2 =   27962025: 0.010011 +/- 0.001453
testing 33554430 bytes of repeated "こんにちは":
                      gcc_strlen =   33554430: 0.019331 +/- 0.000836
                      kjs_strlen =   33554430: 0.035225 +/- 0.000411
                       cp_strlen =   33554430: 0.021429 +/- 0.000309
                 kjs_strlen_utf8 =   11184810: 0.070249 +/- 0.000312
                  gp_strlen_utf8 =   11184810: 0.026545 +/- 0.000621
                  cp_strlen_utf8 =   11184810: 0.050512 +/- 0.000273
             cp_strlen_utf8_sse2 =   11184810: 0.010246 +/- 0.001466
testing 33554416 bytes of repeated "abcdefghijklmnopqrstuvwxyzβ":
                      gcc_strlen =   33554416: 0.019308 +/- 0.001091
                      kjs_strlen =   33554416: 0.035070 +/- 0.000486
                       cp_strlen =   33554416: 0.021441 +/- 0.000289
                 kjs_strlen_utf8 =   32356044: 0.070287 +/- 0.000297
                  gp_strlen_utf8 =   32356044: 0.043681 +/- 0.000429
                  cp_strlen_utf8 =   32356044: 0.050402 +/- 0.000204
             cp_strlen_utf8_sse2 =   32356044: 0.010407 +/- 0.001371