1 ## oils_failures_allowed: 2
2 ## tags: dev-minimal
3
4 #### usage errors
5
6 json read zz
7 echo status=$?
8
9 json write
10
11 ## status: 3
12 ## STDOUT:
13 status=2
14 ## END
15
16 #### json write STRING
17 shopt --set parse_proc
18
19 json write ('foo')
20 var s = 'foo'
21 json write (s)
22 ## STDOUT:
23 "foo"
24 "foo"
25 ## END
26
27 #### json write ARRAY
28 json write (:|foo.cc foo.h|)
29 json write --indent 0 (['foo.cc', 'foo.h'])
30 ## STDOUT:
31 [
32 "foo.cc",
33 "foo.h"
34 ]
35 [
36 "foo.cc",
37 "foo.h"
38 ]
39 ## END
40
41 #### json write Dict
42 json write ({k: 'v', k2: [4, 5]})
43
44 json write ([{k: 'v', k2: 'v2'}, {}])
45
46 ## STDOUT:
47 {
48 "k": "v",
49 "k2": [
50 4,
51 5
52 ]
53 }
54 [
55 {
56 "k": "v",
57 "k2": "v2"
58 },
59 {}
60 ]
61 ## END
62
63 #### json write compact format
64 shopt --set parse_proc
65
66 # TODO: ORDER of keys should be PRESERVED
67 var mydict = {name: "bob", age: 30}
68
69 json write --pretty=0 (mydict)
70 # ignored
71 json write --pretty=F --indent 4 (mydict)
72 ## STDOUT:
73 {"name":"bob","age":30}
74 {"name":"bob","age":30}
75 ## END
76
77 #### json write in command sub
78 shopt -s oil:all # for echo
79 var mydict = {name: "bob", age: 30}
80 json write (mydict)
81 var x = $(json write (mydict))
82 echo $x
83 ## STDOUT:
84 {
85 "name": "bob",
86 "age": 30
87 }
88 {
89 "name": "bob",
90 "age": 30
91 }
92 ## END
93
94 #### json read passed invalid args
95
96 # EOF
97 json read
98 echo status=$?
99
100 json read 'z z'
101 echo status=$?
102
103 json read a b c
104 echo status=$?
105
106 ## STDOUT:
107 status=1
108 status=2
109 status=2
110 ## END
111
112 #### json read uses $_reply var
113
114 # space before true
115 echo ' true' | json read
116 json write (_reply)
117
118 ## STDOUT:
119 true
120 ## END
121
122 #### json read then json write
123
124 # BUG with space before true
125 echo '{"name": "bob", "age": 42, "ok": true}' | json read
126 json write (_reply)
127
128 echo '{"name": "bob", "age": 42, "ok":true}' | json read
129 json write (_reply)
130
131 echo '{"name": {}, "age": {}}' | json read
132 json write (_reply)
133
134 ## STDOUT:
135 {
136 "name": "bob",
137 "age": 42,
138 "ok": true
139 }
140 {
141 "name": "bob",
142 "age": 42,
143 "ok": true
144 }
145 {
146 "name": {},
147 "age": {}
148 }
149 ## END
150
151 #### json read with redirect
152 echo '{"age": 42}' > $TMP/foo.txt
153 json read (&x) < $TMP/foo.txt
154 pp cell :x
155 ## STDOUT:
156 x = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:42)]))
157 ## END
158
159 #### json read at end of pipeline (relies on lastpipe)
160 echo '{"age": 43}' | json read (&y)
161 pp cell y
162 ## STDOUT:
163 y = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:43)]))
164 ## END
165
166 #### invalid JSON
167 echo '{' | json read (&y)
168 echo pipeline status = $?
169 pp line (y)
170 ## status: 1
171 ## STDOUT:
172 pipeline status = 1
173 ## END
174
175 #### Extra data after valid JSON
176
177 # Trailing space is OK
178 echo '42 ' | json read
179 echo num space $?
180
181 echo '{} ' | json read
182 echo obj space $?
183
184 echo '42 # comment' | json8 read
185 echo num comment $?
186
187 echo '{} # comment ' | json8 read
188 echo obj comment $?
189
190 echo '42]' | json read
191 echo num bracket $?
192
193 echo '{}]' | json read
194 echo obj bracket $?
195
196 ## STDOUT:
197 num space 0
198 obj space 0
199 num comment 0
200 obj comment 0
201 num bracket 1
202 obj bracket 1
203 ## END
204
205 #### json write expression
206 json write --pretty=0 ([1,2,3])
207 echo status=$?
208
209 json write (5, 6) # to many args
210 echo status=$?
211
212 ## status: 3
213 ## STDOUT:
214 [1,2,3]
215 status=0
216 ## END
217
218 #### json write evaluation error
219
220 #var block = ^(echo hi)
221 #json write (block)
222 #echo status=$?
223
224 # undefined var
225 json write (a)
226 echo 'should have failed'
227
228 ## status: 1
229 ## STDOUT:
230 ## END
231
232 #### json write of List in cycle
233
234 var L = [1, 2, 3]
235 setvar L[0] = L
236
237 shopt -s ysh:upgrade
238 fopen >tmp.txt {
239 pp line (L)
240 }
241 fgrep -n -o '[ -->' tmp.txt
242
243 json write (L)
244 echo 'should have failed'
245
246 ## status: 1
247 ## STDOUT:
248 1:[ -->
249 ## END
250
251 #### json write of Dict in cycle
252
253 var d = {}
254 setvar d.k = d
255
256 shopt -s ysh:upgrade
257 fopen >tmp.txt {
258 pp line (d)
259 }
260 fgrep -n -o '{ -->' tmp.txt
261
262 json write (d)
263 echo 'should have failed'
264
265 ## status: 1
266 ## STDOUT:
267 1:{ -->
268 ## END
269
270 #### json write of List/Dict referenced twice (bug fix)
271
272 var mylist = [1,2,3]
273 var mydict = {foo: "bar"}
274
275 var top = {k: mylist, k2: mylist, k3: mydict, k4: mydict}
276
277 # BUG!
278 json write --pretty=F (top)
279
280 ## STDOUT:
281 {"k":[1,2,3],"k2":[1,2,3],"k3":{"foo":"bar"},"k4":{"foo":"bar"}}
282 ## END
283
284 #### json read doesn't accept u'' or b'' strings
285
286 json read <<EOF
287 {"key": u'val'}
288 EOF
289 echo status=$?
290
291 #pp line (_reply)
292
293 json read <<EOF
294 {"key": b'val'}
295 EOF
296 echo status=$?
297
298 ## STDOUT:
299 status=1
300 status=1
301 ## END
302
303 #### json read doesn't accept comments, but json8 does
304
305 json8 read <<EOF
306 { # comment
307 "key": # zz
308 b'val', # yy
309 "k2": "v2" #
310 }
311 EOF
312 echo status=$?
313
314 json8 write (_reply)
315
316 json read <<EOF
317 {"key": "val"} # comment
318 EOF
319 echo status=$?
320 ## STDOUT:
321 status=0
322 {
323 "key": "val",
324 "k2": "v2"
325 }
326 status=1
327 ## END
328
329
330 #### json write emits Unicode replacement char for binary data \yff
331
332 json write ([3, "foo", $'-\xff\xfe---\xfd=']) > tmp.txt
333
334 # Round trip it for good measure
335 json read < tmp.txt
336
337 json write (_reply)
338
339 ## STDOUT:
340 [
341 3,
342 "foo",
343 "-��---�="
344 ]
345 ## END
346
347 #### json8 write emits b'' strings for binary data \yff
348
349 json8 write ([3, "foo", $'-\xff\xfe-\xfd='])
350
351 ## STDOUT:
352 [
353 3,
354 "foo",
355 b'-\yff\yfe-\yfd='
356 ]
357 ## END
358
359
360 #### json8 write bytes vs unicode string
361
362 u=$'mu \u03bc \x01 \" \\ \b\f\n\r\t'
363 u2=$'\x01\x1f' # this is a valid unicode string
364
365 b=$'\xff' # this isn't valid unicode
366
367 json8 write (u)
368 json8 write (u2)
369
370 json8 write (b)
371
372 ## STDOUT:
373 "mu μ \u0001 \" \\ \b\f\n\r\t"
374 "\u0001\u001f"
375 b'\yff'
376 ## END
377
378 #### JSON \/ escapes supported
379
380 msg='"\/"'
381
382 echo "$msg" | python3 -c 'import json, sys; print(json.load(sys.stdin))'
383
384 echo "$msg" | json read
385 echo reply=$_reply
386
387 j8="b'\\/'"
388 echo "$msg" | json read
389 echo reply=$_reply
390
391
392 ## STDOUT:
393 /
394 reply=/
395 reply=/
396 ## END
397
398 #### JSON string can have unescaped ' and J8 string can have unescaped "
399
400 json read <<EOF
401 "'"
402 EOF
403
404 pp line (_reply)
405
406 json8 read <<EOF
407 u'"'
408 EOF
409
410 pp line (_reply)
411
412 ## STDOUT:
413 (Str) "'"
414 (Str) "\""
415 ## END
416
417
418 #### J8 supports superfluous \" escapes, but JSON doesn't support \' escapes
419
420 json8 read <<'EOF'
421 b'\"'
422 EOF
423 echo reply=$_reply
424
425 json8 read <<'EOF'
426 b'\'\'\b\f\n\r\t\"\\'
427 EOF
428 pp line (_reply)
429
430 # Suppress traceback
431 python3 -c 'import json, sys; print(json.load(sys.stdin))' 2>/dev/null <<'EOF'
432 "\'"
433 EOF
434 echo python3=$?
435
436 json read <<'EOF'
437 "\'"
438 EOF
439 echo json=$?
440
441 ## STDOUT:
442 reply="
443 (Str) "''\b\f\n\r\t\"\\"
444 python3=1
445 json=1
446 ## END
447
448 #### Escaping uses \u0001 in "", but \u{1} in b''
449
450 s1=$'\x01'
451 s2=$'\x01\xff\x1f' # byte string
452
453 json8 write (s1)
454 json8 write (s2)
455
456 ## STDOUT:
457 "\u0001"
458 b'\u{1}\yff\u{1f}'
459 ## END
460
461
462 #### json8 read
463
464 echo '{ }' | json8 read
465 pp line (_reply)
466
467 echo '[ ]' | json8 read
468 pp line (_reply)
469
470 echo '[42]' | json8 read
471 pp line (_reply)
472
473 echo '[true, false]' | json8 read
474 pp line (_reply)
475
476 echo '{"k": "v"}' | json8 read
477 pp line (_reply)
478
479 echo '{"k": null}' | json8 read
480 pp line (_reply)
481
482 echo '{"k": 1, "k2": 2}' | json8 read
483 pp line (_reply)
484
485 echo "{u'k': {b'k2': null}}" | json8 read
486 pp line (_reply)
487
488 echo '{"k": {"k2": "v2"}, "k3": "backslash \\ \" \n line 2 \u03bc "}' | json8 read
489 pp line (_reply)
490
491 json8 read (&x) <<'EOF'
492 {u'k': {u'k2': u'v2'}, u'k3': u'backslash \\ \" \n line 2 \u{3bc} '}
493 EOF
494 pp line (x)
495
496 ## STDOUT:
497 (Dict) {}
498 (List) []
499 (List) [42]
500 (List) [true,false]
501 (Dict) {"k":"v"}
502 (Dict) {"k":null}
503 (Dict) {"k":1,"k2":2}
504 (Dict) {"k":{"k2":null}}
505 (Dict) {"k":{"k2":"v2"},"k3":"backslash \\ \" \n line 2 μ "}
506 (Dict) {"k":{"k2":"v2"},"k3":"backslash \\ \" \n line 2 μ "}
507 ## END
508
509 #### json8 round trip
510
511 var obj = [42, 1.5, null, true, "hi", b'\yff\yfe\b\n""']
512
513 json8 write --pretty=F (obj) > j
514
515 cat j
516
517 json8 read < j
518
519 json8 write (_reply)
520
521 ## STDOUT:
522 [42,1.5,null,true,"hi",b'\yff\yfe\b\n""']
523 [
524 42,
525 1.5,
526 null,
527 true,
528 "hi",
529 b'\yff\yfe\b\n""'
530 ]
531 ## END
532
533 #### json round trip (regression)
534
535 var d = {
536 short: '-v', long: '--verbose', type: null, default: '', help: 'Enable verbose logging'
537 }
538
539 json write (d) | json read
540
541 pp line (_reply)
542
543 ## STDOUT:
544 (Dict) {"short":"-v","long":"--verbose","type":null,"default":"","help":"Enable verbose logging"}
545 ## END
546
547 #### round trip: decode surrogate pair and encode
548
549 var j = r'"\ud83e\udd26"'
550 echo $j | json read (&c1)
551
552 json write (c1)
553
554 var j = r'"\uD83E\uDD26"'
555 echo $j | json read (&c2)
556
557 json write (c2)
558
559 # Not a surrogate pair
560 var j = r'"\u0001\u0002"'
561 echo $j | json read (&c3)
562
563 json write (c3)
564
565 var j = r'"\u0100\u0101\u0102"'
566 echo $j | json read (&c4)
567
568 json write (c4)
569
570 ## STDOUT:
571 "🤦"
572 "🤦"
573 "\u0001\u0002"
574 "ĀāĂ"
575 ## END
576
577 #### round trip: decode surrogate half and encode
578
579 # TODO: Weird Python allows this to be decoded, but I think the Bjoern state
580 # machine will not!
581
582 shopt -s ysh:upgrade
583
584 for j in '"\ud83e"' '"\udd26"' {
585 var s = fromJson(j)
586 pp line (s)
587
588 # TODO: modify DFA to return the code point in the surrogate range, and
589 # print it in JSON mode
590 # j8 mode could possibly use byte strings
591 json write (s)
592 }
593
594 ## STDOUT:
595 (Str) b'\ya0\ybe'
596 "\ud83e"
597 (Str) b'\yb4\ya6'
598 "\udd26"
599 ## END
600
601 #### toJson() toJson8() - TODO: test difference
602
603 var obj = [42, 1.5, null, true, "hi"]
604
605 echo $[toJson(obj)]
606 echo $[toJson8(obj)]
607
608 ## STDOUT:
609 [42,1.5,null,true,"hi"]
610 [42,1.5,null,true,"hi"]
611 ## END
612
613 #### fromJson() fromJson8() - TODO: test difference
614
615 var message ='[42,1.5,null,true,"hi"]'
616
617 pp line (fromJson(message))
618 pp line (fromJson8(message))
619
620 ## STDOUT:
621 (List) [42,1.5,null,true,"hi"]
622 (List) [42,1.5,null,true,"hi"]
623 ## END
624
625 #### User can handle errors - toJson() toJson8()
626 shopt -s ysh:upgrade
627
628 var obj = []
629 call obj->append(obj)
630
631 try {
632 echo $[toJson(obj)]
633 }
634 echo status=$_status
635 echo "encode error $[_error.message]" | sed 's/0x[a-f0-9]\+/(object id)/'
636
637 try { # use different style
638 echo $[toJson8( /d+/ )]
639 }
640 echo status=$_status
641 echo "encode error $[_error.message]"
642
643 # This makes the interpreter fail with a message
644 echo $[toJson(obj)]
645
646 ## status: 4
647 ## STDOUT:
648 status=4
649 encode error Can't encode List (object id) in object cycle
650 status=4
651 encode error Can't serialize object of type Eggex
652 ## END
653
654 #### User can handle errors - fromJson() fromJson8()
655 shopt -s ysh:upgrade
656
657 var message ='[42,1.5,null,true,"hi"'
658
659 try {
660 var obj = fromJson(message)
661 }
662 echo status=$_status
663 echo "decode error $[_error.message]" | egrep -o '.*Expected.*RBracket'
664
665 try {
666 var obj = fromJson8(message)
667 }
668 echo status=$_status
669 echo "decode error $[_error.message]" | egrep -o '.*Expected.*RBracket'
670
671 try {
672 var obj = fromJson('[+]')
673 }
674 echo "positions $[_error.start_pos] - $[_error.end_pos]"
675
676 # This makes the interpreter fail with a message
677 var obj = fromJson(message)
678
679 ## status: 4
680 ## STDOUT:
681 status=4
682 decode error Expected Id.J8_RBracket
683 status=4
684 decode error Expected Id.J8_RBracket
685 positions 1 - 2
686 ## END
687
688
689 #### ASCII control chars can't appear literally in messages
690 shopt -s ysh:upgrade
691
692 var message=$'"\x01"'
693 #echo $message | od -c
694
695 try {
696 var obj = fromJson(message)
697 }
698 echo status=$_status
699 echo "$[_error.message]" | egrep -o 'ASCII control chars'
700
701 ## STDOUT:
702 status=4
703 ASCII control chars
704 ## END
705
706
707 #### \yff can't appear in u'' code strings (command)
708
709 shopt -s ysh:upgrade
710
711 echo -n b'\yfd' | od -A n -t x1
712 echo -n u'\yfd' | od -A n -t x1
713
714 ## status: 2
715 ## STDOUT:
716 fd
717 ## END
718
719 #### \yff can't appear in u'' code strings (expr)
720
721 var x = b'\yfe'
722 write -n -- $x | od -A n -t x1
723
724 var x = u'\yfe'
725 write -n -- $x | od -A n -t x1
726
727 ## status: 2
728 ## STDOUT:
729 fe
730 ## END
731
732 #### \yff can't appear in u'' multiline code strings
733
734 shopt -s ysh:upgrade
735
736 echo -n b'''\yfc''' | od -A n -t x1
737 echo -n u'''\yfd''' | od -A n -t x1
738
739 ## status: 2
740 ## STDOUT:
741 fc
742 ## END
743
744 #### \yff can't appear in u'' data strings
745
746 #shopt -s ysh:upgrade
747
748 json8 read (&b) <<'EOF'
749 b'\yfe'
750 EOF
751 pp line (b)
752
753 json8 read (&u) <<'EOF'
754 u'\yfe'
755 EOF
756 pp line (u) # undefined
757
758 ## status: 1
759 ## STDOUT:
760 (Str) b'\yfe'
761 ## END
762
763 #### \u{dc00} can't be in surrogate range in code (command)
764
765 shopt -s ysh:upgrade
766
767 echo -n u'\u{dc00}' | od -A n -t x1
768
769 ## status: 2
770 ## STDOUT:
771 ## END
772
773 #### \u{dc00} can't be in surrogate range in code (expr)
774
775 shopt -s ysh:upgrade
776
777 var x = u'\u{dc00}'
778 echo $x | od -A n -t x1
779
780 ## status: 2
781 ## STDOUT:
782 ## END
783
784 #### \u{dc00} can't be in surrogate range in data
785
786 json8 read <<'EOF'
787 ["long string", u'hello \u{d7ff}', "other"]
788 EOF
789 echo status=$?
790
791 json8 read <<'EOF'
792 ["long string", u'hello \u{d800}', "other"]
793 EOF
794 echo status=$?
795
796 json8 read <<'EOF'
797 ["long string", u'hello \u{dfff}', "other"]
798 EOF
799 echo status=$?
800
801 json8 read <<'EOF'
802 ["long string", u'hello \u{e000}', "other"]
803 EOF
804 echo status=$?
805
806
807 ## STDOUT:
808 status=0
809 status=1
810 status=1
811 status=0
812 ## END
813
814
815
816 #### Inf and NaN can't be encoded or decoded
817
818 # This works in Python, should probably support it
819
820 var n = float("NaN")
821 var i = float("inf")
822
823 pp line (n)
824 pp line (i)
825
826 json dump (n)
827 json dump (i)
828
829 ## status: 2
830 ## STDOUT:
831 ## END
832
833 #### Invalid UTF-8 in JSON is rejected
834
835 echo $'"\xff"' | json read
836 echo status=$?
837
838 echo $'"\xff"' | json8 read
839 echo status=$?
840
841 echo $'\xff' | json read
842 echo status=$?
843
844 echo $'\xff' | json8 read
845 echo status=$?
846
847 ## STDOUT:
848 status=1
849 status=1
850 status=1
851 status=1
852 ## END
853
854 #### Invalid JSON in J8 is rejected
855
856 json8 read <<EOF
857 b'$(echo -e -n '\xff')'
858 EOF
859 echo status=$?
860
861 json8 read <<EOF
862 u'$(echo -e -n '\xff')'
863 EOF
864 echo status=$?
865
866 ## STDOUT:
867 status=1
868 status=1
869 ## END
870
871 #### '' means the same thing as u''
872
873 echo "''" | json8 read
874 pp line (_reply)
875
876 echo "'\u{3bc}'" | json8 read
877 pp line (_reply)
878
879 echo "'\yff'" | json8 read
880 echo status=$?
881
882 ## STDOUT:
883 (Str) ""
884 (Str) "μ"
885 status=1
886 ## END
887
888 #### decode deeply nested structure (stack overflow)
889
890 shopt -s ysh:upgrade
891
892 proc pairs(n) {
893 var m = int(n) # TODO: 1 .. n should auto-convert?
894
895 for i in (1 .. m) {
896 write -n -- '['
897 }
898 for i in (1 .. m) {
899 write -n -- ']'
900 }
901 }
902
903 # This is all Python can handle; C++ can handle more
904 msg=$(pairs 50)
905
906 #echo $msg
907
908 echo "$msg" | json read
909 pp line (_reply)
910 echo len=$[len(_reply)]
911
912 ## STDOUT:
913 (List) [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]
914 len=1
915 ## END
916
917 #### decode integer larger than 2^32
918
919 json=$(( 1 << 33 ))
920 echo $json
921
922 echo $json | json read
923 pp line (_reply)
924
925 ## STDOUT:
926 8589934592
927 (Int) 8589934592
928 ## END
929
930 #### round trip: read/write with ysh
931
932 var file = "$REPO_ROOT/spec/testdata/bug.json"
933 #cat $file
934 cat $file | json read (&cfg)
935 json write (cfg) > ysh-json
936
937 diff -u $file ysh-json
938 echo diff=$?
939
940 ## STDOUT:
941 diff=0
942 ## END
943
944 #### round trip: read/write with ysh, read/write with Python 3 (bug regression)
945
946 var file = "$REPO_ROOT/spec/testdata/bug.json"
947 #cat $file
948 cat $file | json read (&cfg)
949 json write (cfg) > ysh-json
950
951 cat ysh-json | python3 -c \
952 'import json, sys; obj = json.load(sys.stdin); json.dump(obj, sys.stdout, indent=2); print()' \
953 > py-json
954
955 diff -u $file py-json
956 echo diff=$?
957
958 ## STDOUT:
959 diff=0
960 ## END
961
962 #### Encoding bytes that don't hit UTF8_REJECT immediately (bug fix)
963
964 var x = $'\xce'
965 json8 write (x)
966 declare -p x
967 echo
968
969 var y = $'\xbc'
970 json8 write (y)
971 declare -p y
972 echo
973
974 var z = $'\xf0\x9f\xa4\xff'
975 json8 write (z)
976 declare -p z
977
978 ## STDOUT:
979 b'\yce'
980 declare -- x=$'\xce'
981
982 b'\ybc'
983 declare -- y=$'\xbc'
984
985 b'\yf0\y9f\ya4\yff'
986 declare -- z=$'\xf0\x9f\xa4\xff'
987 ## END
988
989 #### NIL8 token in JSON / JSON8
990
991 echo "(" | json read
992 echo status=$?
993
994 echo ")" | json8 read
995 echo status=$?
996
997 ## STDOUT:
998 status=1
999 status=1
1000 ## END