1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3/*
4***************************************************************************
5* Copyright (C) 2008-2013, International Business Machines Corporation
6* and others. All Rights Reserved.
7***************************************************************************
8*
9* uspoof_impl.h
10*
11* Implementation header for spoof detection
12*
13*/
14
15#ifndef USPOOFIM_H
16#define USPOOFIM_H
17
18#include "uassert.h"
19#include "unicode/utypes.h"
20#include "unicode/uspoof.h"
21#include "unicode/uscript.h"
22#include "unicode/udata.h"
23#include "udataswp.h"
24#include "utrie2.h"
25
26#if !UCONFIG_NO_NORMALIZATION
27
28#ifdef __cplusplus
29
30#include "capi_helper.h"
31#include "umutex.h"
32
33U_NAMESPACE_BEGIN
34
35// The maximum length (in UTF-16 UChars) of the skeleton replacement string resulting from
36// a single input code point. This is function of the unicode.org data.
37#define USPOOF_MAX_SKELETON_EXPANSION 20
38
39// The default stack buffer size for copies or conversions or normalizations
40// of input strings being checked. (Used in multiple places.)
41#define USPOOF_STACK_BUFFER_SIZE 100
42
43// Magic number for sanity checking spoof data.
44#define USPOOF_MAGIC 0x3845fdef
45
46// Magic number for sanity checking spoof checkers.
47#define USPOOF_CHECK_MAGIC 0x2734ecde
48
49class ScriptSet;
50class SpoofData;
51struct SpoofDataHeader;
52class ConfusableDataUtils;
53
54/**
55 * Class SpoofImpl corresponds directly to the plain C API opaque type
56 * USpoofChecker. One can be cast to the other.
57 */
58class SpoofImpl : public UObject,
59 public IcuCApiHelper<USpoofChecker, SpoofImpl, USPOOF_MAGIC> {
60public:
61 SpoofImpl(SpoofData *data, UErrorCode& status);
62 SpoofImpl(UErrorCode& status);
63 SpoofImpl();
64 void construct(UErrorCode& status);
65 virtual ~SpoofImpl();
66
67 /** Copy constructor, used by the user level uspoof_clone() function.
68 */
69 SpoofImpl(const SpoofImpl &src, UErrorCode &status);
70
71 USpoofChecker *asUSpoofChecker();
72 static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
73 static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);
74
75 /** Set and Get AllowedLocales, implementations of the corresponding API */
76 void setAllowedLocales(const char *localesList, UErrorCode &status);
77 const char * getAllowedLocales(UErrorCode &status);
78
79 // Add (union) to the UnicodeSet all of the characters for the scripts used for
80 // the specified locale. Part of the implementation of setAllowedLocales.
81 void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);
82
83 // Functions implementing the features of UTS 39 section 5.
84 static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status);
85 void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const;
86 void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const;
87 void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const;
88 URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const;
89
90 int32_t findHiddenOverlay(const UnicodeString& input, UErrorCode& status) const;
91 bool isIllegalCombiningDotLeadCharacter(UChar32 cp) const;
92
93 /** parse a hex number. Untility used by the builders. */
94 static UChar32 ScanHex(const char16_t *s, int32_t start, int32_t limit, UErrorCode &status);
95
96 static UClassID U_EXPORT2 getStaticClassID();
97 virtual UClassID getDynamicClassID() const override;
98
99 //
100 // Data Members
101 //
102
103 int32_t fChecks; // Bit vector of checks to perform.
104
105 SpoofData *fSpoofData;
106
107 const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters.
108 // for this Spoof Checker. Defaults to all chars.
109
110 const char *fAllowedLocales; // The list of allowed locales.
111 URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier.
112};
113
114/**
115 * Class CheckResult corresponds directly to the plain C API opaque type
116 * USpoofCheckResult. One can be cast to the other.
117 */
118class CheckResult : public UObject,
119 public IcuCApiHelper<USpoofCheckResult, CheckResult, USPOOF_CHECK_MAGIC> {
120public:
121 CheckResult();
122 virtual ~CheckResult();
123
124 USpoofCheckResult *asUSpoofCheckResult();
125 static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status);
126 static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status);
127
128 void clear();
129
130 // Used to convert this CheckResult to the older int32_t return value API
131 int32_t toCombinedBitmask(int32_t expectedChecks);
132
133 // Data Members
134 int32_t fChecks; // Bit vector of checks that were failed.
135 UnicodeSet fNumerics; // Set of numerics found in the string.
136 URestrictionLevel fRestrictionLevel; // The restriction level of the string.
137};
138
139
140//
141// Confusable Mappings Data Structures, version 2.0
142//
143// For the confusable data, we are essentially implementing a map,
144// key: a code point
145// value: a string. Most commonly one char in length, but can be more.
146//
147// The keys are stored as a sorted array of 32 bit ints.
148// bits 0-23 a code point value
149// bits 24-31 length of value string, in UChars (between 1 and 256 UChars).
150// The key table is sorted in ascending code point order. (not on the
151// 32 bit int value, the flag bits do not participate in the sorting.)
152//
153// Lookup is done by means of a binary search in the key table.
154//
155// The corresponding values are kept in a parallel array of 16 bit ints.
156// If the value string is of length 1, it is literally in the value array.
157// For longer strings, the value array contains an index into the strings table.
158//
159// String Table:
160// The strings table contains all of the value strings (those of length two or greater)
161// concatenated together into one long char16_t (UTF-16) array.
162//
163// There is no nul character or other mark between adjacent strings.
164//
165//----------------------------------------------------------------------------
166//
167// Changes from format version 1 to format version 2:
168// 1) Removal of the whole-script confusable data tables.
169// 2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask.
170// 3) Expansion of string length value in the key bitmask from 2 bits to 8 bits.
171// 4) Removal of the string lengths table since 8 bits is sufficient for the
172// lengths of all entries in confusables.txt.
173
174
175
176// Internal functions for manipulating confusable data table keys
177#define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2 // version for ICU 58
178class ConfusableDataUtils {
179public:
180 inline static UChar32 keyToCodePoint(int32_t key) {
181 return key & 0x00ffffff;
182 }
183 inline static int32_t keyToLength(int32_t key) {
184 return ((key & 0xff000000) >> 24) + 1;
185 }
186 inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) {
187 U_ASSERT((codePoint & 0x00ffffff) == codePoint);
188 U_ASSERT(length <= 256);
189 return codePoint | ((length - 1) << 24);
190 }
191};
192
193
194//-------------------------------------------------------------------------------------
195//
196// SpoofData
197//
198// A small class that wraps the raw (usually memory mapped) spoof data.
199// Serves two primary functions:
200// 1. Convenience. Contains real pointers to the data, to avoid dealing with
201// the offsets in the raw data.
202// 2. Reference counting. When a spoof checker is cloned, the raw data is shared
203// and must be retained until all checkers using the data are closed.
204// Nothing in this struct includes state that is specific to any particular
205// USpoofDetector object.
206//
207//---------------------------------------------------------------------------------------
208class SpoofData: public UMemory {
209 public:
210 static SpoofData* getDefault(UErrorCode &status); // Get standard ICU spoof data.
211 static void releaseDefault(); // Cleanup reference to default spoof data.
212
213 SpoofData(UErrorCode &status); // Create new spoof data wrapper.
214 // Only used when building new data from rules.
215
216 // Constructor for use when creating from prebuilt default data.
217 // A UDataMemory is what the ICU internal data loading functions provide.
218 // The udm is adopted by the SpoofData.
219 SpoofData(UDataMemory *udm, UErrorCode &status);
220
221 // Constructor for use when creating from serialized data.
222 //
223 SpoofData(const void *serializedData, int32_t length, UErrorCode &status);
224
225 // Check raw Spoof Data Version compatibility.
226 // Return true it looks good.
227 UBool validateDataVersion(UErrorCode &status) const;
228
229 ~SpoofData(); // Destructor not normally used.
230 // Use removeReference() instead.
231 // Reference Counting functions.
232 // Clone of a user-level spoof detector increments the ref count on the data.
233 // Close of a user-level spoof detector decrements the ref count.
234 // If the data is owned by us, it will be deleted when count goes to zero.
235 SpoofData *addReference();
236 void removeReference();
237
238 // Reset all fields to an initial state.
239 // Called from the top of all constructors.
240 void reset();
241
242 // Copy this instance's raw data buffer to the specified address.
243 int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const;
244
245 // Get the total number of bytes of data backed by this SpoofData.
246 // Not to be confused with length, which returns the number of confusable entries.
247 int32_t size() const;
248
249 // Get the confusable skeleton transform for a single code point.
250 // The result is a string with a length between 1 and 18 as of Unicode 9.
251 // This is the main public endpoint for this class.
252 // @return The length in UTF-16 code units of the substitution string.
253 int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const;
254
255 // Get the number of confusable entries in this SpoofData.
256 int32_t length() const;
257
258 // Get the code point (key) at the specified index.
259 UChar32 codePointAt(int32_t index) const;
260
261 // Get the confusable skeleton (value) at the specified index.
262 // Append it to the specified UnicodeString&.
263 // @return The length in UTF-16 code units of the skeleton string.
264 int32_t appendValueTo(int32_t index, UnicodeString& dest) const;
265
266 private:
267 // Reserve space in the raw data. For use by builder when putting together a
268 // new set of data. Init the new storage to zero, to prevent inconsistent
269 // results if it is not all otherwise set by the requester.
270 // Return:
271 // pointer to the new space that was added by this function.
272 void *reserveSpace(int32_t numBytes, UErrorCode &status);
273
274 // initialize the pointers from this object to the raw data.
275 void initPtrs(UErrorCode &status);
276
277 SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data
278 UBool fDataOwned; // True if the raw data is owned, and needs
279 // to be deleted when refcount goes to zero.
280 UDataMemory *fUDM; // If not nullptr, our data came from a
281 // UDataMemory, which we must close when
282 // we are done.
283
284 uint32_t fMemLimit; // Limit of available raw data space
285 u_atomic_int32_t fRefCount;
286
287 // Confusable data
288 int32_t *fCFUKeys;
289 uint16_t *fCFUValues;
290 char16_t *fCFUStrings;
291
292 friend class ConfusabledataBuilder;
293};
294
295//---------------------------------------------------------------------------------------
296//
297// Raw Binary Data Formats, as loaded from the ICU data file,
298// or as built by the builder.
299//
300//---------------------------------------------------------------------------------------
301struct SpoofDataHeader {
302 int32_t fMagic; // (0x3845fdef)
303 uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo
304 // if there is one associated with this data.
305 int32_t fLength; // Total length in bytes of this spoof data,
306 // including all sections, not just the header.
307
308 // The following four sections refer to data representing the confusable data
309 // from the Unicode.org data from "confusables.txt"
310
311 int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *)
312 int32_t fCFUKeysSize; // number of entries in keys table (32 bits each)
313
314 // TODO: change name to fCFUValues, for consistency.
315 int32_t fCFUStringIndex; // byte offset to String Indexes table
316 int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each)
317 // (number of entries must be same as in Keys table
318
319 int32_t fCFUStringTable; // byte offset of String table
320 int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars)
321
322 // The following sections are for data from xidmodifications.txt
323
324 int32_t unused[15]; // Padding, Room for Expansion
325};
326
327
328
329U_NAMESPACE_END
330#endif /* __cplusplus */
331
332/**
333 * Endianness swap function for binary spoof data.
334 * @internal
335 */
336U_CAPI int32_t U_EXPORT2
337uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
338 UErrorCode *status);
339
340
341#endif
342
343#endif /* USPOOFIM_H */
344
345