1// © 2022 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3
4#ifndef MLBREAKENGINE_H
5#define MLBREAKENGINE_H
6
7#include "hash.h"
8#include "unicode/resbund.h"
9#include "unicode/uniset.h"
10#include "unicode/utext.h"
11#include "uvectr32.h"
12
13U_NAMESPACE_BEGIN
14
15#if !UCONFIG_NO_BREAK_ITERATION
16
17/**
18 * A machine learning break engine for the phrase breaking in Japanese.
19 */
20class MlBreakEngine : public UMemory {
21 public:
22 /**
23 * Constructor.
24 *
25 * @param digitOrOpenPunctuationOrAlphabetSet An UnicodeSet with the digit, open punctuation and
26 * alphabet.
27 * @param closePunctuationSet An UnicodeSet with close punctuation.
28 * @param status Information on any errors encountered.
29 */
30 MlBreakEngine(const UnicodeSet &digitOrOpenPunctuationOrAlphabetSet,
31 const UnicodeSet &closePunctuationSet, UErrorCode &status);
32
33 /**
34 * Virtual destructor.
35 */
36 virtual ~MlBreakEngine();
37
38 public:
39 /**
40 * Divide up a range of characters handled by this break engine.
41 *
42 * @param inText A UText representing the text
43 * @param rangeStart The start of the range of the characters
44 * @param rangeEnd The end of the range of the characters
45 * @param foundBreaks Output of C array of int32_t break positions, or 0
46 * @param inString The normalized string of text ranging from rangeStart to rangeEnd
47 * @param inputMap The vector storing the native index of inText
48 * @param status Information on any errors encountered.
49 * @return The number of breaks found
50 */
51 int32_t divideUpRange(UText *inText, int32_t rangeStart, int32_t rangeEnd,
52 UVector32 &foundBreaks, const UnicodeString &inString,
53 const LocalPointer<UVector32> &inputMap, UErrorCode &status) const;
54
55 private:
56 /**
57 * Load the machine learning's model file.
58 *
59 * @param error Information on any errors encountered.
60 */
61 void loadMLModel(UErrorCode &error);
62
63 /**
64 * In the machine learning's model file, specify the name of the key and value to load the
65 * corresponding feature and its score.
66 *
67 * @param rb A ResouceBundle corresponding to the model file.
68 * @param keyName The kay name in the model file.
69 * @param valueName The value name in the model file.
70 * @param model A hashtable to store the pairs of the feature and its score.
71 * @param error Information on any errors encountered.
72 */
73 void initKeyValue(UResourceBundle *rb, const char *keyName, const char *valueName,
74 Hashtable &model, UErrorCode &error);
75
76 /**
77 * Initialize the index list from the input string.
78 *
79 * @param inString A input string to be segmented.
80 * @param indexList A code unit index list of inString.
81 * @param status Information on any errors encountered.
82 * @return The number of code units of the first four characters in inString.
83 */
84 int32_t initIndexList(const UnicodeString &inString, int32_t *indexList,
85 UErrorCode &status) const;
86
87 /**
88 * Evaluate whether the index is a potential breakpoint.
89 *
90 * @param inString A input string to be segmented.
91 * @param indexList A code unit index list of the inString.
92 * @param startIdx The start index of the indexList.
93 * @param numCodeUnits The current code unit boundary of the indexList.
94 * @param numBreaks The accumulated number of breakpoints.
95 * @param boundary A vector including the index of the breakpoint.
96 * @param status Information on any errors encountered.
97 * @return The number of breakpoints
98 */
99 int32_t evaluateBreakpoint(const UnicodeString &inString, int32_t *indexList, int32_t startIdx,
100 int32_t numCodeUnits, int32_t numBreaks, UVector32 &boundary,
101 UErrorCode &status) const;
102
103 void printUnicodeString(const UnicodeString &s) const;
104
105 UnicodeSet fDigitOrOpenPunctuationOrAlphabetSet;
106 UnicodeSet fClosePunctuationSet;
107 Hashtable fModel[13]; // {UW1, UW2, ... UW6, BW1, ... BW3, TW1, TW2, ... TW4} 6+3+4= 13
108 int32_t fNegativeSum;
109};
110
111#endif
112
113U_NAMESPACE_END
114
115/* MLBREAKENGINE_H */
116#endif
117