Chapter 11 - Other XML and JSON Technologies

Create sample files

// This creates the sample files used in Chapter 11 for XML.
// This query is #load-ed by other queries, so you don't need to run it directly.
// The files are written to the current (temp) folder, so they're cleaned up when you exit LINQPad

var contacts = @"<?xml version=""1.0"" encoding=""utf-8""?>
<contacts>
  <customer id=""1"">
    <firstname>Sara</firstname>
    <lastname>Wells</lastname>
  </customer>
  <customer id=""2"">
    <firstname>Dylan</firstname>
    <lastname>Lockwood</lastname>
  </customer>
  <supplier>
    <name>Ian Co.</name>
  </supplier>
</contacts>";

if (!File.Exists ("contacts.xml")) File.WriteAllText ("contacts.xml", contacts);

var customer = @"<?xml version=""1.0"" encoding=""utf-8"" standalone=""yes""?>
<customer id=""123"" status=""archived"">
  <firstname>Jim</firstname>
  <lastname>Bo</lastname>
</customer>";

if (!File.Exists ("customer.xml")) File.WriteAllText ("customer.xml", customer);

var customerCredit = @"<?xml version=""1.0"" encoding=""utf-8"" standalone=""yes""?>
<customer id=""123"" status=""archived"">
  <firstname>Jim</firstname>
  <lastname>Bo</lastname>
  <creditlimit>500.00</creditlimit>    <!-- OK, we sneaked this in! -->
</customer>";

if (!File.Exists ("customerCredit.xml")) File.WriteAllText ("customerCredit.xml", customerCredit);

var customerWithCDATA = @"<?xml version=""1.0"" encoding=""utf-8"" ?>
<!DOCTYPE customer [ <!ENTITY tc ""Top Customer""> ]>
<customer id=""123"" status=""archived"">
  <firstname>Jim</firstname>
  <lastname>Bo</lastname>
  <quote><![CDATA[C#'s operators include: < > &]]></quote>
  <notes>Jim Bo is a &tc;</notes>
  <!--  That wasn't so bad! -->
</customer>";

if (!File.Exists ("customerWithCDATA.xml")) File.WriteAllText ("customerWithCDATA.xml", customerWithCDATA);

var customers = @"<?xml version=""1.0"" encoding=""utf-8"" standalone=""yes""?>
<customers>
  <customer id=""123"" status=""archived"">
    <firstname>Jim</firstname>
    <lastname>Bo</lastname>
  </customer>
  <customer id=""125"" status=""archived"">
    <firstname>Todd</firstname>
    <lastname>Bar</lastname>
  </customer>
</customers>";

if (!File.Exists ("customers.xml")) File.WriteAllText ("customers.xml", customers);

var logfile = @"<?xml version=""1.0"" encoding=""utf-8""?>
<log>
<logentry id=""0""><date>2019-10-11T00:00:00-07:00</date><source>test</source></logentry>
<logentry id=""1""><date>2019-10-11T00:00:01-07:00</date><source>test</source></logentry>
<logentry id=""2""><date>2019-10-11T00:00:02-07:00</date><source>test</source></logentry>
<logentry id=""3""><date>2019-10-11T00:00:03-07:00</date><source>test</source></logentry>
<logentry id=""4""><date>2019-10-11T00:00:05-07:00</date><source>test</source></logentry>
<logentry id=""5""><date>2019-10-11T00:00:08-07:00</date><source>test</source></logentry>
<logentry id=""6""><date>2019-10-11T00:00:09-07:00</date><source>test</source></logentry>
<logentry id=""7""><date>2019-10-11T00:00:10-07:00</date><source>test</source></logentry>
<logentry id=""8""><date>2019-10-11T00:00:13-07:00</date><source>test</source></logentry>
<logentry id=""9""><date>2019-10-11T00:00:14-07:00</date><source>test</source></logentry>
</log>";

if (!File.Exists ("logfile.xml")) File.WriteAllText ("logfile.xml", logfile);

var customersSchema = @"<?xml version=""1.0"" encoding=""utf-8""?>
<xs:schema attributeFormDefault=""unqualified""
           elementFormDefault=""qualified""
           xmlns:xs=""http://www.w3.org/2001/XMLSchema"">
  <xs:element name=""customers"">
    <xs:complexType>
      <xs:sequence>
        <xs:element maxOccurs=""unbounded"" name=""customer"">
          <xs:complexType>
            <xs:sequence>
              <xs:element name=""firstname"" type=""xs:string"" />
              <xs:element name=""lastname"" type=""xs:string"" />
            </xs:sequence>
            <xs:attribute name=""id"" type=""xs:int"" use=""required"" />
            <xs:attribute name=""status"" type=""xs:string"" use=""required"" />
          </xs:complexType>
        </xs:element>
      </xs:sequence>
    </xs:complexType>
  </xs:element>
</xs:schema>
";

if (!File.Exists ("customers.xsd")) File.WriteAllText ("customers.xsd", customersSchema);


var customerXslt = @"<?xml version=""1.0"" encoding=""UTF-8""?>
  <xsl:stylesheet xmlns:xsl=""http://www.w3.org/1999/XSL/Transform""
version=""1.0"">
  <xsl:template match=""/"">
    <html>
      <p><xsl:value-of select=""//firstname""/></p>
      <p><xsl:value-of select=""//lastname""/></p>
    </html>
  </xsl:template>
</xsl:stylesheet>
";

if (!File.Exists ("customer.xslt")) File.WriteAllText ("customer.xslt", customerXslt);

Output XML Structure

#load ".\Create sample files.linq"

XmlReaderSettings settings = new XmlReaderSettings();
settings.IgnoreWhitespace = true;

using XmlReader reader = XmlReader.Create ("customer.xml", settings);
while (reader.Read())
{
  Console.Write (new string (' ', reader.Depth * 2));  // Write indentation
  Console.Write (reader.NodeType.ToString());

  if (reader.NodeType == XmlNodeType.Element || reader.NodeType == XmlNodeType.EndElement)
    Console.Write (" Name=" + reader.Name);
  else if (reader.NodeType == XmlNodeType.Text)
    Console.Write (" Value=" + reader.Value);
  
  Console.WriteLine ();
}

EXTRA - Reading Nodes

#load ".\Create sample files.linq"

XmlReaderSettings settings = new XmlReaderSettings();
settings.IgnoreWhitespace = true;
settings.DtdProcessing = DtdProcessing.Parse;    // Required to read DTDs

using XmlReader r = XmlReader.Create ("customerWithCDATA.xml", settings);
while (r.Read())
{
  Console.Write (r.NodeType.ToString().PadRight (17, '-'));
  Console.Write ("> ".PadRight (r.Depth * 3));

  switch (r.NodeType)
  {
    case XmlNodeType.Element:
    case XmlNodeType.EndElement:
      Console.WriteLine (r.Name); break;

    case XmlNodeType.Text:
    case XmlNodeType.CDATA:
    case XmlNodeType.Comment:
    case XmlNodeType.XmlDeclaration:
      Console.WriteLine (r.Value); break;

    case XmlNodeType.DocumentType:
      Console.WriteLine (r.Name + " - " + r.Value); break;

    default: break;
  }
}

Read Element Content As

#load ".\Create sample files.linq"

XmlReaderSettings settings = new XmlReaderSettings();
settings.IgnoreWhitespace = true;

using XmlReader r = XmlReader.Create ("customerCredit.xml", settings);

r.MoveToContent();                // Skip over the XML declaration
r.ReadStartElement ("customer");
string firstName = r.ReadElementContentAsString ("firstname", "");
string lastName = r.ReadElementContentAsString ("lastname", "");
decimal creditLimit = r.ReadElementContentAsDecimal ("creditlimit", "");

r.MoveToContent();      // Skip over that pesky comment
r.ReadEndElement();     // Read the closing customer tag

$"{firstName} {lastName} credit limit: {creditLimit}".Dump();

XML Writer

XmlWriterSettings settings = new XmlWriterSettings();
settings.Indent = true;

using XmlWriter writer = XmlWriter.Create ("foo.xml", settings);

writer.WriteStartElement ("customer");
writer.WriteAttributeString ("id", "1");
writer.WriteAttributeString ("status", "archived");

writer.WriteElementString ("firstname", "Jim");
writer.WriteElementString ("lastname", "Bo");
writer.WriteEndElement();

writer.Dispose();
var xml = File.ReadAllText("foo.xml");
xml.Dump();

XML Writer Namespace

XmlWriterSettings settings = new XmlWriterSettings();
settings.Indent = true;

using XmlWriter writer = XmlWriter.Create ("foo.xml", settings);

writer.WriteStartElement ("o", "customer", "http://oreilly.com");
writer.WriteElementString ("o", "firstname", "http://oreilly.com", "Jim");
writer.WriteElementString ("o", "lastname", "http://oreilly.com", "Bo");

writer.WriteEndElement();

writer.Dispose();
var xml = File.ReadAllText("foo.xml");
xml.Dump();

Serlialize to XML

void Main()
{
  SerializeContacts();
  DeserializeContacts().Dump();
}

private void SerializeContacts()
{
  XmlWriterSettings settings = new XmlWriterSettings();
  settings.Indent = true; // To make visual inspection easier

  using XmlWriter writer = XmlWriter.Create ("contacts.xml", settings);

  Contacts cts = new Contacts()
  {
    Customers = new List<Customer>()
    {
      new Customer() { ID = 1, FirstName = "Sara", LastName = "Wells"},
      new Customer() { ID = 2, FirstName = "Dylan", LastName = "Lockwood"}
    },
    Suppliers = new List<Supplier>()
    {
      new Supplier() { Name = "Ian Weemes" }
    }
  };

  writer.WriteStartElement ("contacts");
  cts.WriteXml (writer);
  writer.WriteEndElement();
}

private Contacts DeserializeContacts()
{
  XmlReaderSettings settings = new XmlReaderSettings();
  settings.IgnoreWhitespace = true;
  settings.IgnoreComments = true;
  settings.IgnoreProcessingInstructions = true;

  using XmlReader reader = XmlReader.Create ("contacts.xml", settings);
  reader.MoveToContent();

  var cts = new Contacts();
  cts.ReadXml (reader);

  return cts;
}

public class Contacts
{
  public IList<Customer> Customers = new List<Customer>();
  public IList<Supplier> Suppliers = new List<Supplier>();

  public void ReadXml (XmlReader r)
  {
    bool isEmpty = r.IsEmptyElement;           // This ensures we don't get
    r.ReadStartElement();                      // snookered by an empty
    if (isEmpty) return;                       // <contacts/> element!
    while (r.NodeType == XmlNodeType.Element)
    {
      if (r.Name == Customer.XmlName) Customers.Add (new Customer (r));
      else if (r.Name == Supplier.XmlName) Suppliers.Add (new Supplier (r));
      else
        throw new XmlException ("Unexpected node: " + r.Name);
    }
    r.ReadEndElement();
  }

  public void WriteXml (XmlWriter w)
  {
    foreach (Customer c in Customers)
    {
      w.WriteStartElement (Customer.XmlName);
      c.WriteXml (w);
      w.WriteEndElement();
    }
    foreach (Supplier s in Suppliers)
    {
      w.WriteStartElement (Supplier.XmlName);
      s.WriteXml (w);
      w.WriteEndElement();
    }
  }

}

public class Customer
{
  public const string XmlName = "customer";
  public int? ID;
  public string FirstName, LastName;

  public Customer () { }
  public Customer (XmlReader r) { ReadXml (r); }

  public void ReadXml (XmlReader r)
  {
    if (r.MoveToAttribute ("id")) ID = r.ReadContentAsInt();
    r.ReadStartElement();
    FirstName = r.ReadElementContentAsString ("firstname", "");
    LastName = r.ReadElementContentAsString ("lastname", "");
    r.ReadEndElement();
  }

  public void WriteXml (XmlWriter w)
  {
    if (ID.HasValue) w.WriteAttributeString ("id", "", ID.ToString());
    w.WriteElementString ("firstname", FirstName);
    w.WriteElementString ("lastname", LastName);
  }
}

public class Supplier
{
  public const string XmlName = "supplier";
  public string Name;

  public Supplier () { }
  public Supplier (XmlReader r) { ReadXml (r); }

  public void ReadXml (XmlReader r)
  {
    r.ReadStartElement();
    Name = r.ReadElementContentAsString ("name", "");
    r.ReadEndElement();
  }

  public void WriteXml (XmlWriter w)
  {
    w.WriteElementString ("name", Name);
  }
}

Reading with XElement

#load ".\Create sample files.linq"

XmlReaderSettings settings = new XmlReaderSettings();
settings.IgnoreWhitespace = true;

using XmlReader r = XmlReader.Create ("logfile.xml", settings);

r.ReadStartElement ("log");
while (r.Name == "logentry")
{
  XElement logEntry = (XElement)XNode.ReadFrom (r);
  int id = (int)logEntry.Attribute ("id");
  DateTime date = (DateTime)logEntry.Element ("date");
  string source = (string)logEntry.Element ("source");
  $"{id} {date} {source}".Dump();
}
r.ReadEndElement();

Writing with XElement

XmlWriterSettings settings = new XmlWriterSettings() { Indent = true }; // Otherwise the XML is written as one very long line.
                                                                        // Saves space but makes it more difficult for humans.

using XmlWriter w = XmlWriter.Create ("logfile.xml", settings);

w.WriteStartElement ("log");
for (int i = 0; i < 1000000; i++)
{
  XElement e = new XElement ("logentry",
                   new XAttribute ("id", i),
                   new XElement ("date", DateTime.Today.AddDays (-1)),
                   new XElement ("source", "test"));
  e.WriteTo (w);
}
w.WriteEndElement ();

w.Dispose();
using var reader = File.OpenText("logfile.xml");
for (int i = 0; i < 10; i++) Console.WriteLine (reader.ReadLine());

Validate with XSD

#load ".\Create sample files.linq"

XmlReaderSettings settings = new XmlReaderSettings();
settings.ValidationType = ValidationType.Schema;
settings.Schemas.Add (null, "customers.xsd");

using (XmlReader r = XmlReader.Create ("customers.xml", settings))
  try { while (r.Read()) ; }
  catch (XmlSchemaValidationException ex)
  {
    $"Invalid XML according to schema: {ex.Message}".Dump();
  }
"Finished processing XML".Dump();

Transform with XSLT

#load ".\Create sample files.linq"

XslCompiledTransform transform = new XslCompiledTransform();
transform.Load ("customer.xslt");
transform.Transform ("customer.xml", "customer.xhtml");

File.ReadAllText("customer.xhtml").Dump();

Create sample JSON files

// This creates the sample files used in Chapter 11 for JSON.
// This query is #load-ed by other queries, so you don't need to run it directly.
// The files are cleaned up when you exit LINQPad

var personJson = @"{
  ""FirstName"":""Sara"",
  ""LastName"":""Wells"",
  ""Age"":35,
  ""Friends"":[""Dylan"",""Ian""]
}";

string personPath = "Person.json";
if (!File.Exists ("personPath")) File.WriteAllText (personPath, personJson);

var personArrayJson = @"[
  {
    ""FirstName"":""Sara"",
    ""LastName"":""Wells"",
    ""Age"":35,
    ""Friends"":[""Ian""]
  },
  {
    ""FirstName"":""Ian"",
    ""LastName"":""Weems"",
    ""Age"":42,
    ""Friends"":[""Joe"",""Eric"",""Li""]
  },
  {
    ""FirstName"":""Dylan"",
    ""LastName"":""Lockwood"",
    ""Age"":46,
    ""Friends"":[""Sara"",""Ian""]
  }
]";

string personArrayPath = "PersonArray.json";
if (!File.Exists (personArrayPath)) File.WriteAllText (personArrayPath, personArrayJson);

JSON Reader

#load ".\Create sample JSON files.linq"

byte[] data = File.ReadAllBytes (personPath);
Utf8JsonReader reader = new Utf8JsonReader (data);
while (reader.Read())
{
  switch (reader.TokenType)
  {
    case JsonTokenType.StartObject:
      Console.WriteLine ($"Start of object");
      break;
    case JsonTokenType.EndObject:
      Console.WriteLine ($"End of object");
      break;
    case JsonTokenType.StartArray:
      Console.WriteLine();
      Console.WriteLine ($"Start of array");
      break;
    case JsonTokenType.EndArray:
      Console.WriteLine ($"End of array");
      break;
    case JsonTokenType.PropertyName:
      Console.Write ($"Property: {reader.GetString()}");
      break;
    case JsonTokenType.String:
      Console.WriteLine ($" Value: {reader.GetString()}");
      break;
    case JsonTokenType.Number:
      Console.WriteLine ($" Value: {reader.GetInt32()}");
      break;
    default:
      Console.WriteLine ($"No support for {reader.TokenType}");
      break;
  }
}

JSON Writer

var options = new JsonWriterOptions { Indented = true };

using (var stream = File.Create ("MyFile.json"))
using (var writer = new Utf8JsonWriter (stream, options))
{
  writer.WriteStartObject();
  // Property name and value specified in one call
  writer.WriteString ("FirstName", "Dylan");
  writer.WriteString ("LastName", "Lockwood");
  // Property name and value specified in separate calls
  writer.WritePropertyName ("Age");
  writer.WriteNumberValue (46);
  writer.WriteCommentValue ("This is a (non-standard) comment");
  writer.WriteEndObject();
}

File.ReadAllText("MyFile.json").Dump();

JsonDocument

Number();
Array();
Age();

void Number()
{
  using JsonDocument document = JsonDocument.Parse ("123");
  JsonElement root = document.RootElement;
  Console.WriteLine (root.ValueKind);       // Number

  int number = document.RootElement.GetInt32();
  Console.WriteLine (number);                // 123
}

void Array()
{
  using JsonDocument document = JsonDocument.Parse (@"[1, 2, 3, 4, 5]").Dump ("Array");
  int length = document.RootElement.GetArrayLength();   // 5
  int value = document.RootElement [3].GetInt32();      // 4
  
  Console.WriteLine($"length: {length}; value {value}");
}

void Age()
{
  using JsonDocument document = JsonDocument.Parse (@"{ ""Age"": 32}").Dump ("Object");
  JsonElement root = document.RootElement;
  int age = root.GetProperty ("Age").GetInt32();
  Console.WriteLine(age);

  // Discover Age property
  JsonProperty ageProp = root.EnumerateObject().First();
  string name = ageProp.Name;             // Age  
  JsonElement value = ageProp.Value;
  Console.WriteLine (value.ValueKind);    // Number
  Console.WriteLine (value.GetInt32());   // 32

}

JsonDocument - with LINQ

#load ".\Create sample JSON files.linq"

using var json = File.OpenRead (personArrayPath);
using JsonDocument document = JsonDocument.Parse (json);

var query =
  from person in document.RootElement.EnumerateArray()
  select new
  {
    FirstName = person.GetProperty ("FirstName").GetString(),
    Age = person.GetProperty ("Age").GetInt32(),
    Friends =
      from friend in person.GetProperty ("Friends").EnumerateArray()
      select friend.GetString()
  };

query.Dump();

JsonDocument - Updating with a JSON writer

#load ".\Create sample JSON files.linq"

using JsonDocument document = JsonDocument.Parse (personArrayJson);

var options = new JsonWriterOptions { Indented = true };

using (var stream = File.Create ("MyFile.json"))
using (var writer = new Utf8JsonWriter (stream, options))
{
  writer.WriteStartArray();
  foreach (var person in document.RootElement.EnumerateArray())
  {
    int friendCount = person.GetProperty ("Friends").GetArrayLength();
    if (friendCount >= 2)
      person.WriteTo (writer);
  }
}

File.ReadAllText ("MyFile.json").Dump("Updated");

JsonNode - Simple parsing

var node = JsonNode.Parse ("123");                  // Parses to a JsonValue
(node is JsonValue).Dump ("node is JsonValue");
node.Dump ("JsonValue");

((JsonValue)node).GetValue<int>().Dump ("number");  // Works, but clumsy!
   node.AsValue().GetValue<int>().Dump ("number");  // Shortcut for above
   node          .GetValue<int>().Dump ("number");  // Better shortcut
                         ((int)node).Dump ("number");  // Even better shortcut!

if (node.AsValue().TryGetValue<int> (out var number))
  number.Dump ("Parse succeeded");

JsonNode - JSON Arrays

var node = JsonNode.Parse (@"[1, 2, 3, 4, 5]").Dump ("JsonArray");
Debug.Assert (node is JsonArray);

node.AsArray().Count.Dump ("Array element count");

foreach (JsonNode child in node.AsArray())
  child.ToJsonString().Dump ("array element");

// Reach directly in to first element via indexer:
Console.WriteLine ((int)node [0]);   // 1

JsonNode - JSON Objects

var node = JsonNode.Parse (@"{ ""Name"":""Alice"", ""Age"": 32}").Dump ("JsonObject");
Debug.Assert (node is JsonObject);

string name = (string)node ["Name"];   // Alice
int age = (int)node ["Age"];           // 32

new { name, age }.Dump();

// Enumerate over the dictionary’s key/value pairs:
foreach (KeyValuePair<string, JsonNode> keyValuePair in node.AsObject())
{
  string propertyName = keyValuePair.Key;
  JsonNode value = keyValuePair.Value;
  new { propertyName, value }.Dump();
}

if (node.AsObject().TryGetPropertyValue ("Name", out JsonNode nameNode))
{
  nameNode.Dump ("JsonObject has a Name property");
}

JsonNode - fluent traversal

#load ".\Create sample JSON files.linq"

JsonNode node = JsonNode.Parse (personArrayJson);

string li = (string) node[1]["Friends"][2];
li.Dump();

JsonNode - LINQ

#load ".\Create sample JSON files.linq"

JsonNode node = JsonNode.Parse (personArrayJson);

var query =
  from person in node.AsArray()
  select new
  {
    FirstName = (string) person ["FirstName"],
    Age = (int) person ["Age"],
    Friends =
      from friend in person ["Friends"].AsArray()
      select (string) friend
  };

query.Dump();

JsonNode - Updating the DOM

var node = JsonNode.Parse ("{ \"Color\": \"Red\" }").Dump ("Before");
node ["Color"] = "White";
node ["Valid"] = true;
node.Dump ("After updates");

node.AsObject().Remove ("Valid");
node.Dump ("After removing property");

var arrayNode = JsonNode.Parse ("[1, 2, 3]").Dump ("arrayNode before");
arrayNode.AsArray().RemoveAt (0);
arrayNode.AsArray().Add (4);
arrayNode.Dump ("arrayNode after");

JsonNode - Updating the DOM - more examples

#load ".\Create sample JSON files.linq"

JsonNode node = JsonNode.Parse (personArrayJson);
string oldJson = node.ToString();

node[0]["Friends"].AsArray().Add ("Amy");   // Give the first person another friend

node[1]["NewValue"] = 123.456;              // Add a new simply-typed property to the second person 

node[2]["NewObject"] = new JsonObject       // Add a new object-typed property to the third person
{
  ["X"] = 1,
  ["Y"] = 2
};

// Write the udpated DOM back to a JSON string:
string newJson = node.ToString();

// Get LINQPad to display the difference between the old and new JSON:
Util.Dif (oldJson, newJson).Dump();

JsonNode - Constructing a DOM programmatically

var node = new JsonArray
{
  new JsonObject {
    ["Name"] = "Tracy",
    ["Age"] = 30,
    ["Friends"] = new JsonArray ("Lisa", "Joe")
  },
  new JsonObject {
    ["Name"] = "Jordyn",
    ["Age"] = 25,
    ["Friends"] = new JsonArray ("Tracy", "Li")
  }
};

node.Dump();