1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.zip.Checksum;
24
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.util.Bytes;
27 import org.apache.hadoop.hbase.util.ChecksumType;
28
29
30
31
32 public class ChecksumUtil {
33
34
35 private static byte[] DUMMY_VALUE = new byte[128 * HFileBlock.CHECKSUM_SIZE];
36
37
38
39
40
41
42
43
44 private static boolean generateExceptions = false;
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60 static void generateChecksums(byte[] indata,
61 int startOffset, int endOffset,
62 byte[] outdata, int outOffset,
63 ChecksumType checksumType,
64 int bytesPerChecksum) throws IOException {
65
66 if (checksumType == ChecksumType.NULL) {
67 return;
68 }
69
70 Checksum checksum = checksumType.getChecksumObject();
71 int bytesLeft = endOffset - startOffset;
72 int chunkNum = 0;
73
74 while (bytesLeft > 0) {
75
76 checksum.reset();
77 int count = Math.min(bytesLeft, bytesPerChecksum);
78 checksum.update(indata, startOffset, count);
79
80
81 int cksumValue = (int)checksum.getValue();
82 outOffset = Bytes.putInt(outdata, outOffset, cksumValue);
83 chunkNum++;
84 startOffset += count;
85 bytesLeft -= count;
86 }
87 }
88
89
90
91
92
93
94
95
96
97
98 static boolean validateBlockChecksum(Path path, HFileBlock block,
99 byte[] data, int hdrSize) throws IOException {
100
101
102
103
104
105
106
107 if (!block.getHFileContext().isUseHBaseChecksum()) {
108 return false;
109 }
110
111
112
113
114
115 ChecksumType cktype = ChecksumType.codeToType(block.getChecksumType());
116 if (cktype == ChecksumType.NULL) {
117 return true;
118 }
119 Checksum checksumObject = cktype.getChecksumObject();
120 checksumObject.reset();
121
122
123 int bytesPerChecksum = block.getBytesPerChecksum();
124
125
126 if (bytesPerChecksum < hdrSize) {
127 String msg = "Unsupported value of bytesPerChecksum. " +
128 " Minimum is " + hdrSize +
129 " but the configured value is " + bytesPerChecksum;
130 HFile.LOG.warn(msg);
131 return false;
132 }
133
134 ByteBuffer hdr = block.getBufferWithHeader();
135 checksumObject.update(hdr.array(), hdr.arrayOffset(), hdrSize);
136
137 int off = hdrSize;
138 int consumed = hdrSize;
139 int bytesLeft = block.getOnDiskDataSizeWithHeader() - off;
140 int cksumOffset = block.getOnDiskDataSizeWithHeader();
141
142
143 while (bytesLeft > 0) {
144 int thisChunkSize = bytesPerChecksum - consumed;
145 int count = Math.min(bytesLeft, thisChunkSize);
146 checksumObject.update(data, off, count);
147
148 int storedChecksum = Bytes.toInt(data, cksumOffset);
149 if (storedChecksum != (int)checksumObject.getValue()) {
150 String msg = "File " + path +
151 " Stored checksum value of " + storedChecksum +
152 " at offset " + cksumOffset +
153 " does not match computed checksum " +
154 checksumObject.getValue() +
155 ", total data size " + data.length +
156 " Checksum data range offset " + off + " len " + count +
157 HFileBlock.toStringHeader(block.getBufferReadOnly());
158 HFile.LOG.warn(msg);
159 if (generateExceptions) {
160 throw new IOException(msg);
161 } else {
162 return false;
163 }
164 }
165 cksumOffset += HFileBlock.CHECKSUM_SIZE;
166 bytesLeft -= count;
167 off += count;
168 consumed = 0;
169 checksumObject.reset();
170 }
171 return true;
172 }
173
174
175
176
177
178
179
180
181 static long numBytes(long datasize, int bytesPerChecksum) {
182 return numChunks(datasize, bytesPerChecksum) *
183 HFileBlock.CHECKSUM_SIZE;
184 }
185
186
187
188
189
190
191
192
193 static long numChunks(long datasize, int bytesPerChecksum) {
194 long numChunks = datasize/bytesPerChecksum;
195 if (datasize % bytesPerChecksum != 0) {
196 numChunks++;
197 }
198 return numChunks;
199 }
200
201
202
203
204
205
206
207
208
209 static void reserveSpaceForChecksums(ByteArrayOutputStream baos,
210 int numBytes, int bytesPerChecksum) throws IOException {
211 long numChunks = numChunks(numBytes, bytesPerChecksum);
212 long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE;
213 while (bytesLeft > 0) {
214 long count = Math.min(bytesLeft, DUMMY_VALUE.length);
215 baos.write(DUMMY_VALUE, 0, (int)count);
216 bytesLeft -= count;
217 }
218 }
219
220
221
222
223
224
225
226 public static void generateExceptionForChecksumFailureForTest(boolean value) {
227 generateExceptions = value;
228 }
229 }
230