Download

Reference Manual

Navigation

← Previous Changeset
Next Changeset →

Changeset 3236

Timestamp:

02/21/08 03:18:34 (10 months ago)

Author:

kris

Message:

increased parsing throughput

Files:

trunk/example/text/xmldom.d (modified) (3 diffs)
trunk/example/text/xmlpull.d (modified) (4 diffs)
trunk/example/text/xmlsax.d (modified) (5 diffs)
trunk/tango/text/xml/Document.d (modified) (7 diffs)
trunk/tango/text/xml/PullParser.d (modified) (7 diffs)
trunk/tango/text/xml/Sax.d (modified) (2 diffs)
trunk/tango/text/xml/XmlIterator.d (modified) (1 diff)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

trunk/example/text/xmldom.d

r3234	r3236
8	8	*******************************************************************************/
9	9
10		~~import tango.io.File;~~
11	10	import tango.io.Stdout;
12	11	import tango.time.StopWatch;
…	…
17	16	*******************************************************************************/
18	17
19		void bench (int iterations~~, char[] filename~~)
	18	void bench (int iterations)
20	19	{
21	20	StopWatch elapsed;
22	21
23	22	auto doc = new Document!(char);
24		auto content = ~~cast(char[]) File(filename).read~~;
	23	auto content = import ("hamlet.xml");
25	24
26	25	elapsed.start;
27		for (auto i=0; ++i < iterations;))
	26	for (auto i=0; ++i < iterations;)
28	27	doc.parse (content);
29	28
…	…
37	36	void main()
38	37	{
39		for (int i=10; ~~--i~~;)
40		bench (~~5000, "soap_mid.xml"~~);
	38	for (int i=10; i--;)
	39	bench (2000);
41	40	}
42	41

trunk/example/text/xmlpull.d

r3234	r3236
1		~~import tango.io.File;~~
2	1	import tango.io.Stdout;
3	2	import tango.time.StopWatch;
…	…
7	6	void benchmark (int iterations, char[] filename)
8	7	{
9		~~char c1;~~
10		~~static char c;~~
11	8	StopWatch elapsed;
12	9
13		auto file = new File (filename);
14		auto content = cast(char[]) file.read;
	10	auto content = import ("hamlet.xml");
15	11	auto parser = new PullParser!(char) (content);
16	12
…	…
18	14	for (auto i=0; ++i < iterations;)
19	15	{
20		while (parser.next)
21		{}
	16	while (parser.next) {}
22	17	parser.reset;
23	18	}
24
25	19	Stdout.formatln ("{} MB/s", (content.length * iterations) / (elapsed.stop * (1024 * 1024)));
26	20	}
…	…
29	23	{
30	24	for (int i = 10; --i;)
31		benchmark (~~5000, "soap_mid~~.xml");
	25	benchmark (2000, "hamlet.xml");
32	26	}

trunk/example/text/xmlsax.d

r3235	r3236
1	1	module xmlsax;
2	2
3		~~import tango.io.File;~~
4	3	import tango.io.Stdout;
5	4	import tango.time.StopWatch;
…	…
7	6	import tango.text.xml.Sax;
8	7
9		void benchmark (int iterations, ~~char[] filename~~)
	8	void benchmark (int iterations, SaxParser!(char) parser, char[] content)
10	9	{
11		~~char c1;~~
12		~~static char c;~~
13	10	StopWatch elapsed;
	11	elapsed.start;
14	12
15		~~auto file = new File (filename);~~
16		~~auto content = cast(char[]) file.read;~~
17		~~auto parser = new SaxParser!(char);~~
18		~~// auto handler = new EventsHandler!(char);~~
19		~~auto handler = new LengthHandler!(char);~~
20		~~parser.setContentHandler(handler);~~
21		~~parser.setContent(content);~~
22
23		~~elapsed.start;~~
24	13	for (auto i=0; ++i < iterations;)
25	14	{
…	…
31	20	}
32	21
	22
33	23	void main()
34	24	{
	25	auto content = import ("hamlet.xml");
	26	auto parser = new SaxParser!(char);
	27	auto handler = new EventsHandler!(char);
	28	//auto handler = new LengthHandler!(char);
	29	parser.setContentHandler(handler);
	30	parser.setContent(content);
	31
35	32	for (int i = 10; --i;)
36		~~benchmark (5000, "soap_mid.xml"~~);
	33	benchmark (2000, parser, content);
37	34	}
	35
	36
	37
38	38
39	39	private class EventsHandler(Ch = char) : ContentHandler!(Ch) {
…	…
64	64	public void startElement(Ch[] uri, Ch[] localName, Ch[] qName, Attribute!(Ch)[] atts) {
65	65	events++;
66		foreach (~~Attribute!(Ch)~~ attr; atts) {
	66	foreach (inout attr; atts) {
67	67	events++;
68	68	}
…	…
122	122	elm++;
123	123	elmlen += localName.length;
124		foreach (~~Attribute!(Ch)~~ attr; atts) {
	124	foreach (inout attr; atts) {
125	125	att++;
126	126	attlen += attr.localName.length;

trunk/tango/text/xml/Document.d

r3226	r3236
12	12
13	13	package import tango.text.xml.PullParser;
	14
	15	version = NameSpace;
14	16
15	17	/*******************************************************************************
…	…
154	156	inscopeNSs["xmlns"] = 2;
155	157
156		while (~~super.next~~)
	158	while (true)
157	159	{
158		switch (super.~~type~~)
	160	switch (super.next)
159	161	{
160	162	case XmlTokenType.EndElement:
…	…
192	194	}
193	195	cur = node;
194
	196	version (NameSpace)
	197	{
195	198	if (node.prefix.length)
196	199	{
…	…
207	210	else
208	211	node.uriID = defNamespace;
	212	}
209	213	break;
210	214
…	…
215	219	attr.localName = super.localName;
216	220	attr.type = XmlNodeType.Attribute;
217
	221	version (NameSpace)
	222	{
218	223	if (super.prefix.length)
219	224	{
…	…
253	258	else
254	259	cur.attrib (attr, defNamespace);
	260	}
	261	else
	262	cur.attrib (attr, defNamespace);
255	263	break;
256	264
…	…
271	279	break;
272	280
	281	case XmlTokenType.Done:
	282	return;
	283
273	284	default:
274	285	break;

trunk/tango/text/xml/PullParser.d

</

r3206	r3236
26	26	*******************************************************************************/
27	27
28		public enum XmlTokenType {StartElement, Attribute, EndElement,
	28	public enum XmlTokenType {Done, StartElement, Attribute, EndElement,
29	29	EndEmptyElement, Data, Comment, CData,
30	30	Doctype, PI, None};
…	…
58	58	private bool err;
59	59	private char[] errMsg;
60		private static Object dummy;
61
62		/***********************************************************************
63
64		Adding this static-ctor gains another 20MB/s
65
66		Go figure ...
67
68		***********************************************************************/
69
70		static this()
71		{
72		dummy = null;
73		}
74
	60
75	61	/***********************************************************************
76	62
…	…
86	72	***********************************************************************/
87	73
88		~~private bool doAttributeName~~()
89		{
	74	final XmlTokenType next()
	75	{
90	76	auto p = text.point;
91		auto q = text.eatAttrName (p);
92
93		if (*q == ':')
94		{
95		prefix = p[0 .. q - p];
96		q = text.eatAttrName (p = q + 1);
97		localName = p[0 .. q - p];
98		}
99		else
100		{
101		prefix = null;
102		localName = p[0 .. q - p];
103		}
104
105		type = XmlTokenType.Attribute;
106		if (*q <= 32)
107		{
108		auto e = text.end;
109		do {
110		if (++q >= e)
111		return doEndOfStream;
112		} while (*q <= 32);
113		}
114
115		if (*q is '=')
116		return doAttributeValue (q + 1);
117		return false;
118		}
119
120		/***********************************************************************
121
122		***********************************************************************/
123
124		private bool doEndEmptyElement()
125		{
126		if (text[0..2] != "/>")
127		return doUnexpected("/>");
	77	if (*p <= 32)
	78	{
	79	while (*++p <= 32)
	80	if (p >= text.end)
	81	return doEndOfStream;
	82	text.point = p;
	83	}
	84
	85	if (type >= XmlTokenType.EndElement)
	86	return doMain;
	87
	88	// in element
	89	switch (*p)
	90	{
	91	case '/':
	92	return doEndEmptyElement;
	93
	94	case '>':
	95	++depth;
	96	++text.point;
	97	return doMain;
	98
	99	default:
	100	return doAttributeName;
	101	}
	102	}
128	103
129		type = XmlTokenType.EndEmptyElement;
130		localName = prefix = null;
131		text.point += 2;
132		return true;
133		}
134
135		/***********************************************************************
136
137		***********************************************************************/
138
139		private bool doComment()
140		{
141		auto p = text.point;
142
143		while (text.good)
144		{
145		if (! text.forwardLocate('-'))
146		return doUnexpectedEOF;
147
148		if (text[0..3] == "-->")
149		{
150		rawValue = p [0 .. text.point - p];
151		type = XmlTokenType.Comment;
152		//prefix = null;
153		text.point += 3;
154		return true;
155		}
156		++text.point;
157		}
158
159		return doUnexpectedEOF;
160		}
161
162		/***********************************************************************
163
164		***********************************************************************/
165
166		private bool doCData()
167		{
168		auto p = text.point;
169
170		while (text.good)
171		{
172		if (! text.forwardLocate(']'))
173		return doUnexpectedEOF;
174
175		if (text[0..3] == "]]>")
176		{
177		type = XmlTokenType.CData;
178		rawValue = p [0 .. text.point - p];
179		//prefix = null;
180		text.point += 3;
181		return true;
182		}
183		++text.point;
184		}
185
186		return doUnexpectedEOF;
187		}
188
189		/***********************************************************************
190
191		***********************************************************************/
192
193		private bool doPI()
194		{
195		auto p = text.point;
196		text.eatElemName;
197		++text.point;
198
199		while (text.good)
200		{
201		if (! text.forwardLocate('\?'))
202		return doUnexpectedEOF;
203
204		if (text.point[1] == '>')
205		{
206		type = XmlTokenType.PI;
207		rawValue = p [0 .. text.point - p];
208		text.point += 2;
209		return true;
210		}
211		++text.point;
212		}
213		return doUnexpectedEOF;
214		}
215
216		/***********************************************************************
217
218		***********************************************************************/
219
220		private bool doDoctype()
221		{
222		text.eatSpace;
223		auto p = text.point;
224
225		while (text.good)
226		{
227		if (*text.point == '>')
228		{
229		type = XmlTokenType.Doctype;
230		rawValue = p [0 .. text.point - p];
231		prefix = null;
232		++text.point;
233		return true;
234		}
235		else
236		if (*text.point == '[')
237		{
238		++text.point;
239		text.forwardLocate(']');
240		++text.point;
241		}
242		else
243		++text.point;
244		}
245
246		if (! text.good)
247		return doUnexpectedEOF;
248		return true;
249		}
250
251		/***********************************************************************
252
253		***********************************************************************/
254
255		private bool doUnexpectedEOF()
256		{
257		return error ("Unexpected EOF");
258		}
259
260		/***********************************************************************
261
262		***********************************************************************/
263
264		private bool doUnexpected(char[] msg = null)
265		{
266		return error ("Unexpected event " ~ msg ~ " " ~ Integer.toString(type));
267		}
268
269		/***********************************************************************
270
271		***********************************************************************/
272
273		private bool doEndOfStream()
274		{
275		return false;
276		}
277
278		/***********************************************************************
279
280		***********************************************************************/
281
282		private bool doMain()
	104	/***********************************************************************
	105
	106	***********************************************************************/
	107
	108	private XmlTokenType doMain()
283	109	{
284	110	auto p = text.point;
…	…
286	112	{
287	113	auto q = p;
288		while (~~++p < text.end~~)
289		~~if (*p is '<')~~
290		{
291		~~type = XmlTokenType.Data;~~
292		rawValue = q [0 .. p - q];
293		text.point = p;
294		~~return true~~;
295		}
296		return ~~doUnexpectedEOF~~;
	114	while (*++p != '<')
	115	{}
	116	if (p < text.end)
	117	{
	118	rawValue = q [0 .. p - q];
	119	text.point = p;
	120	return type = XmlTokenType.Data;
	121	}
	122	return XmlTokenType.Done;
297	123	}
298	124
299	125	switch (p[1])
300	126	{
301		~~case '\?':~~
302		~~text.point += 2;~~
303		~~return doPI();~~
304
305	127	default:
306	128	auto q = ++p;
307		~~//auto e = text.end;~~
308	129	while (q < text.end)
309	130	{
…	…
329	150	}
330	151
331		type = XmlTokenType.StartElement;
332		return true;
	152	return type = XmlTokenType.StartElement;
333	153
334	154	case '!':
…	…
352	172	return doUnexpected("!");
353	173
	174	case '\?':
	175	text.point += 2;
	176	return doPI();
	177
354	178	case '/':
355	179	p += 2;
356	180	auto q = p;
357		auto e = text.end;
358		while (q < e)
359		{
360		auto c = *q;
361		if (c > 63 \|\| text.name[c])
	181	while (q > 63 \|\| text.name[q])
	182	++q;
	183
	184	if (*q is ':')
	185	{
	186	prefix = p[0 .. q - p];
	187	p = ++q;
	188	while (q > 63 \|\| text.attributeName[q])
362	189	++q;
363		else
364		break;
365		}
366		text.point = q;
367
368		if (*q != ':')
	190	localName = p[0 .. q - p];
	191	}
	192	else
369	193	{
370	194	prefix = null;
371	195	localName = p[0 .. q - p];
372	196	}
373		else
374		{
375		prefix = p[0 .. q - p];
376		p = ++text.point;
377		q = text.eatAttrName;
378		localName = p[0 .. q - p];
379		}
380
381		auto end = text.end;
382		while (*q <= 32 && q <= end)
	197
	198	while (*q <= 32)
383	199	++q;
384	200
385		type = XmlTokenType.EndElement;
386		if (*q == '>')
	201	if (q >= text.end)
	202	return doUnexpectedEOF;
	203
	204	if (*q is '>')
387	205	{
388	206	text.point = q + 1;
389	207	--depth;
390		return t~~rue~~;
	208	return type = XmlTokenType.EndElement;
391	209	}
392	210	return doUnexpected(">");
393	211	}
394	212
395		return ~~fals~~e;
396		}
397
398		/***********************************************************************
399
400		***********************************************************************/
401
402		~~final bool next~~()
403		{
	213	return XmlTokenType.Done;
	214	}
	215
	216	/***********************************************************************
	217
	218	***********************************************************************/
	219
	220	private XmlTokenType doAttributeName()
	221	{
404	222	auto p = text.point;
405		if (*p <= 32)
406		{
407		while (*++p <= 32)
408		if (p >= text.end)
409		return doEndOfStream;
410		text.point = p;
	223	auto q = p;
	224	auto e = text.end;
	225
	226	while (q > 63 \|\| text.attributeName[q])
	227	++q;
	228	if (q >= e)
	229	return doUnexpectedEOF;
	230
	231	if (*q is ':')
	232	{
	233	prefix = p[0 .. q - p];
	234	p = ++q;
	235
	236	while (q > 63 \|\| text.attributeName[q])
	237	++q;
	238	if (q >= e)
	239	return doUnexpectedEOF;
	240
	241	localName = p[0 .. q - p];
	242	}
	243	else
	244	{
	245	prefix = null;
	246	localName = p[0 .. q - p];
411	247	}
412	248
413		if (type >= XmlTokenType.EndElement)
414		return doMain;
415
416		// in element
417		switch (*p)
418		{
419		case '/':
420		return doEndEmptyElement;
421
422		case '>':
423		++depth;
	249	if (*q <= 32)
	250	{
	251	while (*++q <= 32) {}
	252	if (q >= e)
	253	return doUnexpectedEOF;
	254	}
	255
	256	if (*q is '=')
	257	{
	258	while (*++q <= 32) {}
	259	if (q >= e)
	260	return doUnexpectedEOF;
	261
	262	auto quote = *q;
	263	switch (quote)
	264	{
	265	case '"':
	266	case '\'':
	267	p = q + 1;
	268	while (*++q != quote) {}
	269	//q = text.forwardLocate(p, quote);
	270	if (q < e)
	271	{
	272	rawValue = p[0 .. q - p];
	273	text.point = q + 1; //Skip end quote
	274	return type = XmlTokenType.Attribute;
	275	}
	276	return doUnexpectedEOF;
	277
	278	default:
	279	return doUnexpected("\' or \"");
	280	}
	281	}
	282
	283	return doUnexpected (q[0..1]);
	284	}
	285
	286	/***********************************************************************
	287
	288	***********************************************************************/
	289
	290	private XmlTokenType doEndEmptyElement()
	291	{
	292	if (text.point[0] is '/' && text.point[1] is '>')
	293	{
	294	localName = prefix = null;
	295	text.point += 2;
	296	return type = XmlTokenType.EndEmptyElement;
	297	}
	298	return doUnexpected("/>");
	299	}
	300
	301	/***********************************************************************
	302
	303	***********************************************************************/
	304
	305	private XmlTokenType doComment()
	306	{
	307	auto p = text.point;
	308
	309	while (text.good)
	310	{
	311	if (! text.forwardLocate('-'))
	312	return doUnexpectedEOF;
	313
	314	if (text[0..3] == "-->")
	315	{
	316	rawValue = p [0 .. text.point - p];
	317	//prefix = null;
	318	text.point += 3;
	319	return type = XmlTokenType.Comment;
	320	}
	321	++text.point;
	322	}
	323
	324	return doUnexpectedEOF;
	325	}
	326
	327	/***********************************************************************
	328
	329	***********************************************************************/
	330
	331	private XmlTokenType doCData()
	332	{
	333	auto p = text.point;
	334
	335	while (text.good)
	336	{
	337	if (! text.forwardLocate(']'))
	338	return doUnexpectedEOF;
	339
	340	if (text[0..3] == "]]>")
	341	{
	342	rawValue = p [0 .. text.point - p];
	343	//prefix = null;
	344	text.point += 3;
	345	return type = XmlTokenType.CData;
	346	}
	347	++text.point;
	348	}
	349
	350	return doUnexpectedEOF;
	351	}
	352
	353	/***********************************************************************
	354
	355	***********************************************************************/
	356
	357	private XmlTokenType doPI()
	358	{
	359	auto p = text.point;
	360	text.eatElemName;
	361	++text.point;
	362
	363	while (text.good)
	364	{
	365	if (! text.forwardLocate('\?'))
	366	return doUnexpectedEOF;
	367
	368	if (text.point[1] == '>')
	369	{
	370	rawValue = p [0 .. text.point - p];
	371	text.point += 2;
	372	return type = XmlTokenType.PI;
	373	}
	374	++text.point;
	375	}
	376	return doUnexpectedEOF;
	377	}
	378
	379	/***********************************************************************
	380
	381	***********************************************************************/
	382
	383	private XmlTokenType doDoctype()
	384	{
	385	text.eatSpace;
	386	auto p = text.point;
	387
	388	while (text.good)
	389	{
	390	if (*text.point == '>')
	391	{
	392	rawValue = p [0 .. text.point - p];
	393	prefix = null;
	394	++text.point;
	395	return type = XmlTokenType.Doctype;
	396	}
	397	else
	398	if (*text.point == '[')
	399	{
424	400	++text.point;
425		return doMain;
426
427		default:
428		break;
429		}
430		return doAttributeName;
431		}
432
433		/***********************************************************************
434
435		***********************************************************************/
436
437		private bool doAttributeValue(Ch* q)
438		{
439		auto p = text.eatSpace (q);
440		auto quote = *p++;
441
442		switch (quote)
443		{
444		case '"':
445		case '\'':
446		q = text.forwardLocate(p, quote);
447		rawValue = p[0 .. q - p];
448		text.point = q + 1; //Skip end quote
449		return true;
450
451		default:
452		return doUnexpected("\' or \"");
453		}
454		}
455
456		/***********************************************************************
457
458		***********************************************************************/
459
460		private bool error (char[] msg)
	401	text.forwardLocate(']');
	402	++text.point;
	403	}
	404	else
	405	++text.point;
	406	}
	407
	408	if (! text.good)
	409	return doUnexpectedEOF;
	410	return XmlTokenType.Doctype;
	411	}
	412
	413	/***********************************************************************
	414
	415	***********************************************************************/
	416
	417	private XmlTokenType doUnexpectedEOF()
	418	{
	419	return error ("Unexpected EOF");
	420	}
	421
	422	/***********************************************************************
	423
	424	***********************************************************************/
	425
	426	private XmlTokenType doUnexpected(char[] msg = null)
	427	{
	428	return error ("Unexpected event " ~ msg ~ " " ~ Integer.toString(type));
	429	}
	430
	431	/***********************************************************************
	432
	433	***********************************************************************/
	434
	435	private XmlTokenType doEndOfStream()
	436	{
	437	return XmlTokenType.Done;
	438	}
	439
	440	/***********************************************************************