PullParser.d

Revision 4065, 27.1 kB (checked in by kris, 2 months ago)

fixed issue with slicing at the end of the document. This was highlighted via a comment at the end of a doc, such as in this example:

Property svn:eol-style set to native

Line
1	/*******************************************************************************
2
3	Copyright: Copyright (C) 2007 Aaron Craelius and Kris Bell
4	All rights reserved.
5
6	License: BSD style: $(LICENSE)
7
8	version: Initial release: February 2008
9
10	Authors: Aaron, Kris
11
12	*******************************************************************************/
13
14	module tango.text.xml.PullParser;
15
16	private import tango.text.Util : indexOf;
17
18	private import tango.core.Exception : XmlException;
19
20	private import Integer = tango.text.convert.Integer;
21
22	version = d;
23
24	/*******************************************************************************
25
26	*******************************************************************************/
27
28	public enum XmlNodeType {Element, Data, Attribute, CData,
29	Comment, PI, Doctype, Document};
30
31	/*******************************************************************************
32
33	*******************************************************************************/
34
35	public enum XmlTokenType {Done, StartElement, Attribute, EndElement,
36	EndEmptyElement, Data, Comment, CData,
37	Doctype, PI, None};
38
39
40	/*******************************************************************************
41
42	Token based XML Parser. Templated to operate with char[], wchar[],
43	and dchar[] based Xml strings.
44
45	The parser is constructed with some tradeoffs relating to document
46	integrity. It is generally optimized for well-formed documents, and
47	currently may read past a document-end for those that are not well
48	formed. There are various compilation options to enable checks and
49	balances, depending on how things should be handled. We'll settle
50	on a common configuration over the next few weeks, but for now all
51	settings are somewhat experimental. Partly because making some tiny
52	unrelated change to the code can cause notable throughput changes,
53	and we need to track that down.
54
55	We're not yet clear why these swings are so pronounced (for changes
56	outside the code path) but they seem to be related to the alignment
57	of codegen. It could be a cache-line issue, or something else. We'll
58	figure it out, yet it's interesting that some hardware buttons are
59	clearly being pushed
60
61	*******************************************************************************/
62
63	class PullParser(Ch = char)
64	{
65	public int depth;
66	public Ch[] prefix;
67	public Ch[] rawValue;
68	public Ch[] localName;
69	public XmlTokenType type = XmlTokenType.None;
70
71	package XmlIterator!(Ch) text;
72	private bool err;
73	private char[] errMsg;
74
75	/***********************************************************************
76
77	***********************************************************************/
78
79	this(Ch[] content = null)
80	{
81	reset (content);
82	}
83
84	/***********************************************************************
85
86	***********************************************************************/
87
88	final XmlTokenType next()
89	{
90	auto p = text.point;
91	if (*p <= 32)
92	{
93	while (*++p <= 32)
94	if (p >= text.end)
95	return endOfInput;
96	text.point = p;
97	}
98
99	if (type >= XmlTokenType.EndElement)
100	return doMain;
101
102	// in element
103	switch (*p)
104	{
105	case '/':
106	return doEndEmptyElement;
107
108	case '>':
109	++depth;
110	++text.point;
111	return doMain;
112
113	default:
114	break;
115	}
116	return doAttributeName;
117	}
118
119	/***********************************************************************
120
121	***********************************************************************/
122
123	private XmlTokenType doMain()
124	{
125	auto e = text.end;
126	auto p = text.point;
127
128	if (*p != '<')
129	{
130	auto q = p;
131	if (p < e)
132	version(d){
133	while (*++p != '<' && p != e) {}
134	}else{
135	while (*++p != '<') {}
136	}
137	if (p < e)
138	{
139	rawValue = q [0 .. p - q];
140	text.point = p;
141	return type = XmlTokenType.Data;
142	}
143	return endOfInput;
144	}
145
146	switch (p[1])
147	{
148	default:
149	// scan new element name
150	auto q = ++p;
151	while (q > 63 \|\| text.name[q])
152	++q;
153
154	// check if we ran past the end
155	if (q >= e)
156	return endOfInput;
157
158	text.point = q;
159	if (*q != ':')
160	{
161	prefix = null;
162	localName = p [0 .. q - p];
163	}
164	else
165	{
166	version (c)
167	{
168	prefix = p[0 .. q - p];
169	p = ++q;
170	while (q > 63 \|\| text.attributeName[q])
171	++q;
172	localName = p[0 .. q - p];
173	text.point = q;
174	}
175	else
176	{
177	prefix = p [0 .. q - p];
178	p = ++text.point;
179	q = text.eatAttrName;
180	localName = p [0 .. q - p];
181	}
182	}
183	return type = XmlTokenType.StartElement;
184
185	case '!':
186	if (text[2..4] == "--")
187	{
188	text.point += 4;
189	return doComment;
190	}
191	else
192	if (text[2..9] == "[CDATA[")
193	{
194	text.point += 9;
195	return doCData;
196	}
197	else
198	if (text[2..9] == "DOCTYPE")
199	{
200	text.point += 9;
201	return doDoctype;
202	}
203	return doUnexpected("!", p);
204
205	case '\?':
206	text.point += 2;
207	return doPI;
208
209	case '/':
210	p += 2;
211	auto q = p;
212	while (q > 63 \|\| text.name[q])
213	++q;
214
215	if (*q is ':')
216	{
217	prefix = p[0 .. q - p];
218	p = ++q;
219	while (q > 63 \|\| text.attributeName[q])
220	++q;
221
222	localName = p[0 .. q - p];
223	}
224	else
225	{
226	prefix = null;
227	localName = p[0 .. q - p];
228	}
229
230	while (*q <= 32)
231	++q;
232
233	if (q >= e)
234	return endOfInput;
235
236	if (*q is '>')
237	{
238	text.point = q + 1;
239	--depth;
240	return type = XmlTokenType.EndElement;
241	}
242	return doExpected(">", q);
243	}
244	}
245
246	/***********************************************************************
247
248	***********************************************************************/
249
250	private XmlTokenType doAttributeName()
251	{
252	auto p = text.point;
253	auto q = p;
254	auto e = text.end;
255
256	while (q > 63 \|\| text.attributeName[q])
257	++q;
258	if (q >= e)
259	return endOfInput;
260
261	if (*q is ':')
262	{
263	prefix = p[0 .. q - p];
264	p = ++q;
265
266	while (q > 63 \|\| text.attributeName[q])
267	++q;
268
269	localName = p[0 .. q - p];
270	}
271	else
272	{
273	prefix = null;
274	localName = p[0 .. q - p];
275	}
276
277	if (*q <= 32)
278	{
279	while (*++q <= 32) {}
280	if (q >= e)
281	return endOfInput;
282	}
283
284	if (*q is '=')
285	{
286	while (*++q <= 32) {}
287	if (q >= e)
288	return endOfInput;
289
290	auto quote = *q;
291	switch (quote)
292	{
293	case '"':
294	case '\'':
295	p = q + 1;
296	while (*++q != quote) {}
297	//q = text.forwardLocate(p, quote);
298	if (q < e)
299	{
300	rawValue = p[0 .. q - p];
301	text.point = q + 1; //Skip end quote
302	return type = XmlTokenType.Attribute;
303	}
304	return endOfInput;
305
306	default:
307	return doExpected("\' or \"", q);
308	}
309	}
310
311	return doUnexpected (q[0..1], q);
312	}
313
314	/***********************************************************************
315
316	***********************************************************************/
317
318	private XmlTokenType doEndEmptyElement()
319	{
320	if (text.point[0] is '/' && text.point[1] is '>')
321	{
322	localName = prefix = null;
323	text.point += 2;
324	return type = XmlTokenType.EndEmptyElement;
325	}
326	return doExpected("/>", text.point);
327	}
328
329	/***********************************************************************
330
331	***********************************************************************/
332
333	private XmlTokenType doComment()
334	{
335	auto p = text.point;
336
337	while (text.good)
338	{
339	if (! text.forwardLocate('-'))
340	return endOfInput;
341
342	if (text[0..3] == "-->")
343	{
344	rawValue = p [0 .. text.point - p];
345	//prefix = null;
346	text.point += 3;
347	return type = XmlTokenType.Comment;
348	}
349	++text.point;
350	}
351
352	return endOfInput;
353	}
354
355	/***********************************************************************
356
357	***********************************************************************/
358
359	private XmlTokenType doCData()
360	{
361	auto p = text.point;
362
363	while (text.good)
364	{
365	if (! text.forwardLocate(']'))
366	return endOfInput;
367
368	if (text[0..3] == "]]>")
369	{
370	rawValue = p [0 .. text.point - p];
371	//prefix = null;
372	text.point += 3;
373	return type = XmlTokenType.CData;
374	}
375	++text.point;
376	}
377
378	return endOfInput;
379	}
380
381	/***********************************************************************
382
383	***********************************************************************/
384
385	private XmlTokenType doPI()
386	{
387	auto p = text.point;
388	text.eatElemName;
389	++text.point;
390
391	while (text.good)
392	{
393	if (! text.forwardLocate('\?'))
394	return endOfInput;
395
396	if (text.point[1] == '>')
397	{
398	rawValue = p [0 .. text.point - p];
399	text.point += 2;
400	return type = XmlTokenType.PI;
401	}
402	++text.point;
403	}
404	return endOfInput;
405	}
406
407	/***********************************************************************
408
409	***********************************************************************/
410
411	private XmlTokenType doDoctype()
412	{
413	text.eatSpace;
414	auto p = text.point;
415
416	while (text.good)
417	{
418	if (*text.point == '>')
419	{
420	rawValue = p [0 .. text.point - p];
421	prefix = null;
422	++text.point;
423	return type = XmlTokenType.Doctype;
424	}
425	else
426	if (*text.point == '[')
427	{
428	++text.point;
429	text.forwardLocate(']');
430	++text.point;
431	}
432	else
433	++text.point;
434	}
435
436	if (! text.good)
437	return endOfInput;
438	return XmlTokenType.Doctype;
439	}
440
441	/***********************************************************************
442
443	***********************************************************************/
444
445	private XmlTokenType endOfInput ()
446	{
447	if (depth)
448	error ("Unexpected EOF");
449
450	return XmlTokenType.Done;
451	}
452
453	/***********************************************************************
454
455	***********************************************************************/
456
457	private XmlTokenType doUnexpected (char[] msg, Ch* p)
458	{
459	return position ("parse error :: unexpected " ~ msg, p);
460	}
461
462	/***********************************************************************
463
464	***********************************************************************/
465
466	private XmlTokenType doExpected (char[] msg, Ch* p)
467	{
468	return position ("parse error :: expected " ~ msg ~ " instead of " ~ *p, p);
469	}
470
471	/***********************************************************************
472
473	***********************************************************************/
474
475	private XmlTokenType position (char[] msg, Ch* p)
476	{
477	return error (msg ~ " at position " ~ Integer.toString(p-text.text.ptr));
478	}
479
480	/***********************************************************************
481
482	***********************************************************************/
483
484	private XmlTokenType error (char[] msg)
485	{
486	err = true;
487	errMsg = msg;
488	throw new XmlException (msg);
489	}
490
491	/***********************************************************************
492
493	***********************************************************************/
494
495	final Ch[] value()
496	{
497	return rawValue;
498	}
499
500	/***********************************************************************
501
502	***********************************************************************/
503
504	final Ch[] name()
505	{
506	if (prefix.length)
507	return prefix ~ ":" ~ localName;
508	return localName;
509	}
510
511	/***********************************************************************
512
513	***********************************************************************/
514
515	final bool error()
516	{
517	return err;
518	}
519
520	/***********************************************************************
521
522	***********************************************************************/
523
524	final bool reset()
525	{
526	text.seek (0);
527	reset_;
528	return true;
529	}
530
531	/***********************************************************************
532
533	***********************************************************************/
534
535	final void reset(Ch[] newText)
536	{
537	text.reset (newText);
538	reset_;
539	}
540
541	/***********************************************************************
542
543	***********************************************************************/
544
545	private void reset_()
546	{
547	err = false;
548	depth = 0;
549	type = XmlTokenType.None;
550
551	if (text.point)
552	{
553	static if (Ch.sizeof == 1)
554	{
555	//Read UTF8 BOM
556	if (*text.point == 0xef)
557	{
558	if (text.point[1] == 0xbb)
559	{
560	if(text.point[2] == 0xbf)
561	text.point += 3;
562	}
563	}
564	}
565
566	//TODO enable optional declaration parsing
567	text.eatSpace;
568	if (*text.point == '<')
569	{
570	if (text.point[1] == '\?')
571	{
572	if (text[2..5] == "xml")
573	{
574	text.point += 5;
575	text.forwardLocate('\?');
576	text.point += 2;
577	}
578	}
579	}
580	}
581	}
582	}
583
584
585	/*******************************************************************************
586
587	*******************************************************************************/
588
589	private struct XmlIterator(Ch)
590	{
591	package Ch* end;
592	package size_t len;
593	package Ch[] text;
594	package Ch* point;
595
596	final bool good()