1/*
2 * Copyright 2012-present Facebook, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <folly/experimental/io/HugePages.h>
18
19#include <fcntl.h>
20#include <sys/stat.h>
21#include <sys/types.h>
22
23#include <cctype>
24#include <cstring>
25
26#include <algorithm>
27#include <stdexcept>
28#include <system_error>
29
30#include <boost/regex.hpp>
31
32#include <folly/Conv.h>
33#include <folly/CppAttributes.h>
34#include <folly/Format.h>
35#include <folly/Range.h>
36#include <folly/String.h>
37
38#include <folly/gen/Base.h>
39#include <folly/gen/File.h>
40#include <folly/gen/String.h>
41
42namespace folly {
43
44namespace {
45
46// Get the default huge page size
47size_t getDefaultHugePageSize() {
48 // We need to parse /proc/meminfo
49 static const boost::regex regex(R"!(Hugepagesize:\s*(\d+)\s*kB)!");
50 size_t pageSize = 0;
51 boost::cmatch match;
52
53 bool error = gen::byLine("/proc/meminfo") | [&](StringPiece line) -> bool {
54 if (boost::regex_match(line.begin(), line.end(), match, regex)) {
55 StringPiece numStr(
56 line.begin() + match.position(1), size_t(match.length(1)));
57 pageSize = to<size_t>(numStr) * 1024; // in KiB
58 return false; // stop
59 }
60 return true;
61 };
62
63 if (error) {
64 throw std::runtime_error("Can't find default huge page size");
65 }
66 return pageSize;
67}
68
69// Get raw huge page sizes (without mount points, they'll be filled later)
70HugePageSizeVec readRawHugePageSizes() {
71 // We need to parse file names from /sys/kernel/mm/hugepages
72 static const boost::regex regex(R"!(hugepages-(\d+)kB)!");
73 boost::smatch match;
74 HugePageSizeVec vec;
75 fs::path path("/sys/kernel/mm/hugepages");
76 for (fs::directory_iterator it(path); it != fs::directory_iterator(); ++it) {
77 std::string filename(it->path().filename().string());
78 if (boost::regex_match(filename, match, regex)) {
79 StringPiece numStr(
80 filename.data() + match.position(1), size_t(match.length(1)));
81 vec.emplace_back(to<size_t>(numStr) * 1024);
82 }
83 }
84 return vec;
85}
86
87// Parse the value of a pagesize mount option
88// Format: number, optional K/M/G/T suffix, trailing junk allowed
89size_t parsePageSizeValue(StringPiece value) {
90 static const boost::regex regex(R"!((\d+)([kmgt])?.*)!", boost::regex::icase);
91 boost::cmatch match;
92 if (!boost::regex_match(value.begin(), value.end(), match, regex)) {
93 throw std::runtime_error("Invalid pagesize option");
94 }
95 char c = '\0';
96 if (match.length(2) != 0) {
97 c = char(tolower(value[size_t(match.position(2))]));
98 }
99 StringPiece numStr(value.data() + match.position(1), size_t(match.length(1)));
100 auto const size = to<size_t>(numStr);
101 auto const mult = [c] {
102 switch (c) {
103 case 't':
104 return 1ull << 40;
105 case 'g':
106 return 1ull << 30;
107 case 'm':
108 return 1ull << 20;
109 case 'k':
110 return 1ull << 10;
111 default:
112 return 1ull << 0;
113 }
114 }();
115 return size * mult;
116}
117
118/**
119 * Get list of supported huge page sizes and their mount points, if
120 * hugetlbfs file systems are mounted for those sizes.
121 */
122HugePageSizeVec readHugePageSizes() {
123 HugePageSizeVec sizeVec = readRawHugePageSizes();
124 if (sizeVec.empty()) {
125 return sizeVec; // nothing to do
126 }
127 std::sort(sizeVec.begin(), sizeVec.end());
128
129 size_t defaultHugePageSize = getDefaultHugePageSize();
130
131 struct PageSizeLess {
132 bool operator()(const HugePageSize& a, size_t b) const {
133 return a.size < b;
134 }
135 bool operator()(size_t a, const HugePageSize& b) const {
136 return a < b.size;
137 }
138 };
139
140 // Read and parse /proc/mounts
141 std::vector<StringPiece> parts;
142 std::vector<StringPiece> options;
143
144 gen::byLine("/proc/mounts") | gen::eachAs<StringPiece>() |
145 [&](StringPiece line) {
146 parts.clear();
147 split(" ", line, parts);
148 // device path fstype options uid gid
149 if (parts.size() != 6) {
150 throw std::runtime_error("Invalid /proc/mounts line");
151 }
152 if (parts[2] != "hugetlbfs") {
153 return; // we only care about hugetlbfs
154 }
155
156 options.clear();
157 split(",", parts[3], options);
158 size_t pageSize = defaultHugePageSize;
159 // Search for the "pagesize" option, which must have a value
160 for (auto& option : options) {
161 // key=value
162 const char* p = static_cast<const char*>(
163 memchr(option.data(), '=', option.size()));
164 if (!p) {
165 continue;
166 }
167 if (StringPiece(option.data(), p) != "pagesize") {
168 continue;
169 }
170 pageSize = parsePageSizeValue(StringPiece(p + 1, option.end()));
171 break;
172 }
173
174 auto pos = std::lower_bound(
175 sizeVec.begin(), sizeVec.end(), pageSize, PageSizeLess());
176 if (pos == sizeVec.end() || pos->size != pageSize) {
177 throw std::runtime_error("Mount page size not found");
178 }
179 if (!pos->mountPoint.empty()) {
180 // Only one mount point per page size is allowed
181 return;
182 }
183
184 // Store mount point
185 fs::path path(parts[1].begin(), parts[1].end());
186 struct stat st;
187 const int ret = stat(path.string().c_str(), &st);
188 if (ret == -1 && errno == ENOENT) {
189 return;
190 }
191 checkUnixError(ret, "stat hugepage mountpoint failed");
192 pos->mountPoint = fs::canonical(path);
193 pos->device = st.st_dev;
194 };
195
196 return sizeVec;
197}
198
199} // namespace
200
201const HugePageSizeVec& getHugePageSizes() {
202 static HugePageSizeVec sizes = readHugePageSizes();
203 return sizes;
204}
205
206const HugePageSize* getHugePageSize(size_t size) {
207 // Linear search is just fine.
208 for (auto& p : getHugePageSizes()) {
209 if (p.mountPoint.empty()) {
210 continue;
211 }
212 if (size == 0 || size == p.size) {
213 return &p;
214 }
215 }
216 return nullptr;
217}
218
219const HugePageSize* getHugePageSizeForDevice(dev_t device) {
220 // Linear search is just fine.
221 for (auto& p : getHugePageSizes()) {
222 if (p.mountPoint.empty()) {
223 continue;
224 }
225 if (device == p.device) {
226 return &p;
227 }
228 }
229 return nullptr;
230}
231
232} // namespace folly
233