Package Martel :: Package test :: Module test_IterParser
[hide private]
[frames] | no frames]

Source Code for Module Martel.test.test_IterParser

  1  import Martel 
  2  from Martel import IterParser, RecordReader, LAX, Parser 
  3   
  4  # Used a lot 
5 -def gen_iterator():
6 return IterParser.IterHeaderFooter( 7 Martel.Re(r"a*\R").make_parser(), 8 RecordReader.CountLines, 9 (1,), 10 11 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(debug_level = 1), 12 RecordReader.CountLines, 13 (2,), 14 15 Martel.Re(r"d*\R").make_parser(), 16 RecordReader.CountLines, 17 (1,), 18 19 "spam")
20
21 -def test_hf1():
22 ip = gen_iterator() 23 24 lines = ["aaaaaaaaa", 25 "b", 26 "c", 27 "bb", 28 "cc", 29 "bbb", 30 "ccc", 31 "d"] 32 text = "\n".join(lines) + "\n" 33 34 35 i = 1 36 for x in ip.iterateString(text): 37 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n" 38 i = i + 1
39
40 -def test_hf2():
41 ip = IterParser.IterHeaderFooter( 42 None, 43 None, 44 None, 45 46 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(), 47 RecordReader.CountLines, 48 (2,), 49 50 Martel.Re(r"d*\R").make_parser(), 51 RecordReader.CountLines, 52 (1,), 53 54 "spam") 55 56 lines = ["b", 57 "c", 58 "bb", 59 "cc", 60 "bbb", 61 "ccc", 62 "d"] 63 text = "\n".join(lines) + "\n" 64 65 i = 1 66 for x in ip.iterateString(text): 67 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n" 68 i = i + 1
69
70 -def test_hf3():
71 ip = IterParser.IterHeaderFooter( 72 None, 73 None, 74 None, 75 76 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(), 77 RecordReader.CountLines, 78 (2,), 79 80 None, 81 None, 82 None, 83 84 "spam") 85 86 lines = ["b", 87 "c", 88 "bb", 89 "cc", 90 "bbb", 91 "ccc", 92 ] 93 text = "\n".join(lines) + "\n" 94 95 i = 1 96 for x in ip.iterateString(text): 97 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n" 98 i = i + 1
99
100 -def test_hf4():
101 ip = IterParser.IterHeaderFooter( 102 Martel.Re(r"a*\R").make_parser(), 103 RecordReader.CountLines, 104 (1,), 105 106 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(), 107 RecordReader.CountLines, 108 (2,), 109 110 None, 111 None, 112 None, 113 114 "spam") 115 116 lines = ["aaaaaaaaa", 117 "b", 118 "c", 119 "bb", 120 "cc", 121 "bbb", 122 "ccc", 123 ] 124 text = "\n".join(lines) + "\n" 125 126 i = 1 127 for x in ip.iterateString(text): 128 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n" 129 i = i + 1
130 131
132 -def test_hf5():
133 # This has a syntax error in the header 134 ip = gen_iterator() 135 136 lines = ["aaaaAaaaa", 137 "b", 138 "c", 139 "bb", 140 "cc", 141 "bbb", 142 "ccc", 143 "d"] 144 text = "\n".join(lines) + "\n" 145 146 147 try: 148 for x in ip.iterateString(text): 149 pass 150 except Parser.ParserPositionException, exc: 151 assert exc.pos == 4
152
153 -def test_hf6():
154 # This has a syntax error in the first line of the first record 155 ip = gen_iterator() 156 157 lines = ["aaaaaaaaa", 158 "-", 159 "c", 160 "bb", 161 "cc", 162 "bbb", 163 "ccc", 164 "d"] 165 text = "\n".join(lines) + "\n" 166 167 168 try: 169 for x in ip.iterateString(text): 170 pass 171 except Parser.ParserPositionException, exc: 172 assert exc.pos == 10
173
174 -def test_hf7():
175 # This has a syntax error in the first line of the second record 176 ip = gen_iterator() 177 178 lines = ["aaaaaaaaa", 179 "b", 180 "c", 181 "b-", 182 "cc", 183 "bbb", 184 "ccc", 185 "d"] 186 text = "\n".join(lines) + "\n" 187 188 189 try: 190 for x in ip.iterateString(text): 191 pass 192 except Parser.ParserPositionException, exc: 193 assert exc.pos == 14, exc.pos
194
195 -def test_hf8():
196 # This has a syntax error in the footer 197 ip = gen_iterator() 198 199 lines = ["aaaaaaaaa", 200 "b", 201 "c", 202 "bb", 203 "cc", 204 "bbb", 205 "ccc", 206 "d-"] 207 text = "\n".join(lines) + "\n" 208 209 210 try: 211 for x in ip.iterateString(text): 212 pass 213 except Parser.ParserPositionException, exc: 214 assert exc.pos == 29, exc.pos
215 216 217 ### the simple record iterator
218 -def test_ri1():
219 ip = IterParser.IterRecords( 220 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(), 221 RecordReader.CountLines, 222 (2,), 223 224 "spam") 225 226 lines = ["b", 227 "c", 228 "bb", 229 "cc", 230 "bbb", 231 "ccc", 232 ] 233 text = "\n".join(lines) + "\n" 234 i = 1 235 for x in ip.iterateString(text): 236 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n" 237 i = i + 1
238 239
240 -def test_ri2():
241 # error in the first record 242 ip = IterParser.IterRecords( 243 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(debug_level = 1), 244 RecordReader.CountLines, 245 (2,), 246 247 "spam") 248 249 lines = ["b", 250 "-", 251 "bb", 252 "cc", 253 "bbb", 254 "ccc", 255 ] 256 text = "\n".join(lines) + "\n" 257 try: 258 for x in ip.iterateString(text): 259 pass 260 except Parser.ParserPositionException, exc: 261 assert exc.pos == 2, exc.pos
262
263 -def test_ri3():
264 # error in the second record 265 ip = IterParser.IterRecords( 266 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(debug_level = 1), 267 RecordReader.CountLines, 268 (2,), 269 270 "spam") 271 272 lines = ["b", 273 "c", 274 "b-", 275 "cc", 276 "bbb", 277 "ccc", 278 ] 279 text = "\n".join(lines) + "\n" 280 try: 281 for x in ip.iterateString(text): 282 pass 283 except Parser.ParserPositionException, exc: 284 assert exc.pos == 5, exc.pos
285
286 -def test_make_iterparsers1():
287 exp = Martel.ParseRecords("dataset", {}, 288 Martel.Group("spam", Martel.Re(r"a*\R")), 289 RecordReader.CountLines, (1,)) 290 iterator = exp.make_iterator("spam") 291 assert isinstance(iterator, IterParser.IterRecords) 292 lines = [] 293 for i in range(0, 10): 294 lines.append("a" * i + "\n") 295 text = "".join(lines) 296 297 i = 0 298 for rec in iterator.iterateString(text): 299 assert len(rec["spam"][0][:-1]) == i, (i, rec["spam"][0]) 300 i = i + 1 301 assert i == 10
302 303
304 -def test_make_iterparsers2():
305 exp = Martel.HeaderFooter("dataset", {}, 306 Martel.Group("header", Martel.Re(r"(a*\R)*")), 307 RecordReader.Until, ("b",), 308 309 Martel.Group("record", Martel.Re(r"(b*\R)*")), 310 RecordReader.Until, ("c",), 311 312 Martel.Group("footer", Martel.Re(r"(c*\R)*")), 313 RecordReader.Everything, (),) 314 315 iterator = exp.make_iterator("record") 316 assert isinstance(iterator, IterParser.IterHeaderFooter), iterator 317 lines = ["a" 318 "aa", 319 "aaaaaaa", 320 "b", 321 "bb", 322 "bbbb", 323 "bbbbbbbb", 324 "bbbbbbbbbbbbbbbb", 325 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 326 "cccc", 327 "cc", 328 "c", 329 ] 330 331 text = "\n".join(lines) + "\n" 332 333 i = 0 334 for rec in iterator.iterateString(text): 335 i = i + 1 336 assert i == 1, i
337
338 -def test_missing_end1():
339 exp = Martel.ParseRecords("dataset", {}, 340 Martel.Group("record", Martel.Re(r"(b*\R)*a\R")), 341 RecordReader.EndsWith, ("a",)) 342 lines = [ 343 "bbb", 344 "bb", 345 "a", 346 "b", 347 "a", 348 "a", 349 ] 350 text = "\n".join(lines) + "\n" 351 352 iterator = exp.make_iterator("record") 353 # This should work 354 for x in iterator.iterateString(text): 355 pass 356 357 # This should not 358 lines.append("c") 359 text = "\n".join(lines) + "\n" 360 try: 361 for x in iterator.iterateString(text): 362 pass 363 raise AssertionError 364 except Parser.ParserPositionException, exc: 365 assert exc.pos == 15, exc.pos
366
367 -def test_missing_end2():
368 # Same as the test_missing_end1 but using HeaderFooter 369 exp = Martel.HeaderFooter("dataset", {}, 370 None, None, None, 371 Martel.Group("record", Martel.Re(r"(b*\R)*a\R")), 372 RecordReader.EndsWith, ("a",), 373 None, None, None 374 ) 375 lines = [ 376 "bbb", 377 "bb", 378 "a", 379 "b", 380 "a", 381 "a", 382 ] 383 text = "\n".join(lines) + "\n" 384 385 iterator = exp.make_iterator("record") 386 # This should work 387 for x in iterator.iterateString(text): 388 pass 389 390 # This should not 391 lines.append("c") 392 text = "\n".join(lines) + "\n" 393 try: 394 for x in iterator.iterateString(text): 395 pass 396 raise AssertionError 397 except Parser.ParserPositionException, exc: 398 assert exc.pos == 15, exc.pos
399
400 -def test_missing_end3():
401 # This one is missing the footer 402 exp = Martel.HeaderFooter("dataset", {}, 403 None, None, None, 404 405 Martel.Group("record", Martel.Re(r"(b*\R)*a\R")), 406 RecordReader.EndsWith, ("a",), 407 408 Martel.Group("footer", Martel.Re(r"c\R")), 409 RecordReader.CountLines, (1,) 410 ) 411 lines = [ 412 "bbb", 413 "bb", 414 "a", 415 "b", 416 "a", 417 "a", 418 "c", # This will be removed for the test 419 ] 420 text = "\n".join(lines) + "\n" 421 422 iterator = exp.make_iterator("record") 423 # This should work 424 for x in iterator.iterateString(text): 425 pass 426 427 # This should not 428 lines.pop() 429 text = "\n".join(lines) + "\n" 430 try: 431 for x in iterator.iterateString(text): 432 pass 433 raise AssertionError 434 except Parser.ParserPositionException, exc: 435 assert exc.pos == 15, exc.pos
436 437
438 -def test():
439 test_hf1() 440 test_hf2() 441 test_hf3() 442 test_hf4() 443 test_hf5() 444 test_hf6() 445 test_hf7() 446 test_hf8() 447 448 test_ri1() 449 test_ri2() 450 test_ri3() 451 452 test_make_iterparsers1() 453 test_make_iterparsers2() 454 455 test_missing_end1() 456 test_missing_end2() 457 test_missing_end3()
458 459 if __name__ == "__main__": 460 test() 461