1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
#!/usr/local/bin/perl
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
use strict;
require "genverifier.pm";
use genverifier;
my(@utf8_cls);
my(@utf8_st);
my($utf8_ver);
#
#
# UTF8 encode the UCS4 into 1 to 4 bytes
#
# 1 byte 00 00 00 00 00 00 00 7f
# 2 bytes 00 00 00 80 00 00 07 ff
# 3 bytes 00 00 08 00 00 00 ff ff
# 4 bytes 00 01 00 00 00 10 ff ff
#
# However, since Surrogate area should not be encoded into UTF8 as
# a Surrogate pair, we can remove the surrogate area from UTF8
#
# 1 byte 00 00 00 00 00 00 00 7f
# 2 bytes 00 00 00 80 00 00 07 ff
# 3 bytes 00 00 08 00 00 00 d7 ff
# 00 00 e0 00 00 00 ff ff
# 4 bytes 00 01 00 00 00 10 ff ff
#
# Now we break them into 6 bits group for 2-4 bytes UTF8
#
# 1 byte 00 7f
# 2 bytes 02 00 1f 3f
# 3 bytes 00 20 00 0d 1f 3f
# 0e 00 00 0f 3f 3f
# 4 bytes 00 10 00 00 04 0f 3f 3f
#
# Break down more
#
# 1 byte 00 7f
# 2 bytes 02 00 1f 3f
# 3 bytes 00 20 00 00 3f 3f
# 01 00 00 0c 3f 3f
# 0d 00 00 0d 1f 3f
# 0e 00 00 0f 3f 3f
# 4 bytes 00 10 00 00 00 3f 3f 3f
# 01 00 00 00 03 3f 3f 3f
# 04 00 00 00 04 0f 3f 3f
#
# Now, add
# c0 to the lead byte of 2 bytes UTF8
# e0 to the lead byte of 3 bytes UTF8
# f0 to the lead byte of 4 bytes UTF8
# 80 to the trail bytes
#
# 1 byte 00 7f
# 2 bytes c2 80 df bf
# 3 bytes e0 a0 80 e0 bf bf
# e1 80 80 ec bf bf
# ed 80 80 ed 9f bf
# ee 80 80 ef bf bf
# 4 bytes f0 90 80 80 f0 bf bf bf
# f1 80 80 80 f3 bf bf bf
# f4 80 80 80 f4 8f bf bf
#
#
# Now we can construct our state diagram
#
# 0:0x0e,0x0f,0x1b->Error
# 0:[0-0x7f]->0
# 0:[c2-df]->3
# 0:e0->4
# 0:[e1-ec, ee-ef]->5
# 0:ed->6
# 0:f0->7
# 0:[f1-f3]->8
# 0:f4->9
# 0:*->Error
# 3:[80-bf]->0
# 3:*->Error
# 4:[a0-bf]->3
# 4:*->Error
# 5:[80-bf]->3
# 5:*->Error
# 6:[80-9f]->3
# 6:*->Error
# 7:[90-bf]->5
# 7:*->Error
# 8:[80-bf]->5
# 8:*->Error
# 9:[80-8f]->5
# 9:*->Error
#
# Now, we classified chars into class
#
# 00,0e,0f,1b:k0
# 01-0d,10-1a,1c-7f:k1
# 80-8f:k2
# 90-9f:k3
# a0-bf:k4
# c0-c1:k0
# c2-df:k5
# e0:k6
# e1-ec:k7
# ed:k8
# ee-ef:k7
# f0:k9
# f1-f3:k10
# f4:k11
# f5-ff:k0
#
# Now, let's put them into array form
@utf8_cls = (
[ 0x00 , 0x00 , 1 ],
[ 0x0e , 0x0f , 0 ],
[ 0x1b , 0x1b , 0 ],
[ 0x01 , 0x0d , 1 ],
[ 0x10 , 0x1a , 1 ],
[ 0x1c , 0x7f , 1 ],
[ 0x80 , 0x8f , 2 ],
[ 0x90 , 0x9f , 3 ],
[ 0xa0 , 0xbf , 4 ],
[ 0xc0 , 0xc1 , 0 ],
[ 0xc2 , 0xdf , 5 ],
[ 0xe0 , 0xe0 , 6 ],
[ 0xe1 , 0xec , 7 ],
[ 0xed , 0xed , 8 ],
[ 0xee , 0xef , 7 ],
[ 0xf0 , 0xf0 , 9 ],
[ 0xf1 , 0xf3 , 10 ],
[ 0xf4 , 0xf4 , 11 ],
[ 0xf5 , 0xff , 0 ],
);
#
# Now, we write the state diagram in class
#
# 0:k0->Error
# 0:k1->0
# 0:k5->3
# 0:k6->4
# 0:k7->5
# 0:k8->6
# 0:k9->7
# 0:k10->8
# 0:k11->9
# 0:*->Error
# 3:k2,k3,k4->0
# 3:*->Error
# 4:k4->3
# 4:*->Error
# 5:k2,k3,k4->3
# 5:*->Error
# 6:k2,k3->3
# 6:*->Error
# 7:k3,k4->5
# 7:*->Error
# 8:k2,k3,k4->5
# 8:*->Error
# 9:k2->5
# 9:*->Error
#
# Now, let's put them into array
#
package genverifier;
@utf8_st = (
# 0 1 2 3 4 5 6 7 8 9 10 11
1, 0, 1, 1, 1, 3, 4, 5, 6, 7, 8, 9, # state 0 Start
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, # state 1 Error
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, # state 2 ItsMe
1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, # state 3
1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, # state 4
1, 1, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, # state 5
1, 1, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, # state 6
1, 1, 5, 5, 1, 1, 1, 1, 1, 1, 1, 1, # state 7
1, 1, 5, 5, 5, 1, 1, 1, 1, 1, 1, 1, # state 8
1, 1, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, # state 9
);
$utf8_ver = genverifier::GenVerifier("UTF8", "UTF-8", \@utf8_cls, 12, \@utf8_st);
print $utf8_ver;
|
