1//
2// Copyright (c) Microsoft. All rights reserved.
3// Licensed under the MIT license. See LICENSE file in the project root for full license information.
4//
5
6//----------------------------------------------------------
7// MethodContextReader.cpp - Abstraction for reading MethodContexts
8// Should eventually support multithreading
9//----------------------------------------------------------
10
11#include "standardpch.h"
12#include "tocfile.h"
13#include "methodcontextreader.h"
14#include "methodcontext.h"
15#include "logging.h"
16#include "runtimedetails.h"
17
18// Just a helper...
19HANDLE MethodContextReader::OpenFile(const char* inputFile, DWORD flags)
20{
21 HANDLE fileHandle =
22 CreateFileA(inputFile, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | flags, NULL);
23 if (fileHandle == INVALID_HANDLE_VALUE)
24 {
25 LogError("Failed to open file '%s'. GetLastError()=%u", inputFile, GetLastError());
26 }
27 return fileHandle;
28}
29
30static std::string to_lower(const std::string& input)
31{
32 std::string res = input;
33 std::transform(input.cbegin(), input.cend(), res.begin(), tolower);
34 return res;
35}
36
37// Looks for a file named foo.origSuffix.newSuffix or foo.newSuffix
38// but only if foo.origSuffix exists.
39//
40// Note: filename extensions must be lower-case, even on case-sensitive file systems!
41std::string MethodContextReader::CheckForPairedFile(const std::string& fileName,
42 const std::string& origSuffix,
43 const std::string& newSuffix)
44{
45 std::string tmp = to_lower(origSuffix);
46
47 // First, check to see if foo.origSuffix exists
48 size_t suffix_offset = fileName.find_last_of('.');
49 if ((SSIZE_T)suffix_offset <= 0 || (tmp != to_lower(fileName.substr(suffix_offset))) ||
50 (GetFileAttributesA(fileName.c_str()) == INVALID_FILE_ATTRIBUTES))
51 return std::string();
52
53 // next, check foo.orig.new from foo.orig
54 tmp = fileName + newSuffix;
55 if (GetFileAttributesA(tmp.c_str()) != INVALID_FILE_ATTRIBUTES)
56 return tmp;
57
58 // Now let's check for foo.new from foo.new.orig
59 tmp = fileName.substr(0, suffix_offset);
60 if (GetFileAttributesA(tmp.c_str()) != INVALID_FILE_ATTRIBUTES)
61 return tmp;
62
63 // Finally, lets try foo.orig from foo.new
64 tmp += newSuffix;
65 if (GetFileAttributesA(tmp.c_str()) != INVALID_FILE_ATTRIBUTES)
66 return tmp;
67
68 return std::string();
69}
70
71MethodContextReader::MethodContextReader(
72 const char* inputFileName, const int* indexes, int indexCount, char* hash, int offset, int increment)
73 : fileHandle(INVALID_HANDLE_VALUE)
74 , fileSize(0)
75 , curMCIndex(0)
76 , Indexes(indexes)
77 , IndexCount(indexCount)
78 , curIndexPos(0)
79 , Hash(hash)
80 , curTOCIndex(0)
81 , Offset(offset)
82 , Increment(increment)
83{
84 this->mutex = CreateMutexA(NULL, FALSE, nullptr);
85
86 std::string tocFileName, mchFileName;
87
88 // First, check to see if they passed an MCH file (look for a paired MCT file)
89 tocFileName = MethodContextReader::CheckForPairedFile(inputFileName, ".mch", ".mct");
90 if (!tocFileName.empty())
91 {
92 mchFileName = inputFileName;
93 }
94 else
95 {
96 // Okay, it wasn't an MCH file, let's check to see if it was an MCT file
97 // so check for a paired MCH file instead
98 mchFileName = MethodContextReader::CheckForPairedFile(inputFileName, ".mct", ".mch");
99 if (!mchFileName.empty())
100 {
101 tocFileName = inputFileName;
102 }
103 else
104 {
105 mchFileName = inputFileName;
106 }
107 }
108
109 if (!tocFileName.empty())
110 this->tocFile.LoadToc(tocFileName.c_str());
111
112 // we'll get here even if we don't have a valid index file
113 this->fileHandle = OpenFile(mchFileName.c_str(), (this->hasTOC() && this->hasIndex()) ? FILE_ATTRIBUTE_NORMAL
114 : FILE_FLAG_SEQUENTIAL_SCAN);
115 if (this->fileHandle != INVALID_HANDLE_VALUE)
116 {
117 GetFileSizeEx(this->fileHandle, (PLARGE_INTEGER) & this->fileSize);
118 }
119
120 ReadExcludedMethods(mchFileName);
121}
122
123MethodContextReader::~MethodContextReader()
124{
125 if (fileHandle != INVALID_HANDLE_VALUE)
126 {
127 CloseHandle(this->fileHandle);
128 }
129
130 CloseHandle(this->mutex);
131
132 CleanExcludedMethods();
133}
134
135bool MethodContextReader::AcquireLock()
136{
137 DWORD res = WaitForSingleObject(this->mutex, INFINITE);
138 return (res == WAIT_OBJECT_0);
139}
140
141void MethodContextReader::ReleaseLock()
142{
143 ReleaseMutex(this->mutex);
144}
145
146bool MethodContextReader::atEof()
147{
148 __int64 pos = 0;
149 SetFilePointerEx(this->fileHandle, *(PLARGE_INTEGER)&pos, (PLARGE_INTEGER)&pos,
150 FILE_CURRENT); // LARGE_INTEGER is a crime against humanity
151 return pos == this->fileSize;
152}
153
154MethodContextBuffer MethodContextReader::ReadMethodContextNoLock(bool justSkip)
155{
156 DWORD bytesRead;
157 char buff[512];
158 unsigned int totalLen = 0;
159 if (atEof())
160 {
161 return MethodContextBuffer();
162 }
163 Assert(ReadFile(this->fileHandle, buff, 2 + sizeof(unsigned int), &bytesRead, NULL) == TRUE);
164 AssertMsg((buff[0] == 'm') && (buff[1] == 'c'), "Didn't find magic number");
165 memcpy(&totalLen, &buff[2], sizeof(unsigned int));
166 if (justSkip)
167 {
168 __int64 pos = totalLen + 2;
169 // Just move the file pointer ahead the correct number of bytes
170 AssertMsg(SetFilePointerEx(this->fileHandle, *(PLARGE_INTEGER)&pos, (PLARGE_INTEGER)&pos, FILE_CURRENT) == TRUE,
171 "SetFilePointerEx failed (Error %X)", GetLastError());
172
173 // Increment curMCIndex as we advanced the file pointer by another MC
174 ++curMCIndex;
175
176 return MethodContextBuffer(0);
177 }
178 else
179 {
180 unsigned char* buff2 = new unsigned char[totalLen + 2]; // total + End Canary
181 Assert(ReadFile(this->fileHandle, buff2, totalLen + 2, &bytesRead, NULL) == TRUE);
182
183 // Increment curMCIndex as we read another MC
184 ++curMCIndex;
185
186 return MethodContextBuffer(buff2, totalLen);
187 }
188}
189
190MethodContextBuffer MethodContextReader::ReadMethodContext(bool acquireLock, bool justSkip)
191{
192 if (acquireLock && !this->AcquireLock())
193 {
194 LogError("Can't acquire the reader lock!");
195 return MethodContextBuffer(-1);
196 }
197
198 struct Param
199 {
200 MethodContextReader* pThis;
201 MethodContextBuffer ret;
202 bool justSkip;
203 } param;
204 param.pThis = this;
205 param.ret = MethodContextBuffer(-2);
206 param.justSkip = justSkip;
207
208 PAL_TRY(Param*, pParam, &param)
209 {
210 pParam->ret = pParam->pThis->ReadMethodContextNoLock(pParam->justSkip);
211 }
212 PAL_FINALLY
213 {
214 this->ReleaseLock();
215 }
216 PAL_ENDTRY
217
218 return param.ret;
219}
220
221// Read a method context buffer from the ContextCollection
222// (either a hive [single] or an index)
223MethodContextBuffer MethodContextReader::GetNextMethodContext()
224{
225 struct Param : FilterSuperPMIExceptionsParam_CaptureException
226 {
227 MethodContextReader* pThis;
228 MethodContextBuffer mcb;
229 } param;
230 param.pThis = this;
231
232 PAL_TRY(Param*, pParam, &param)
233 {
234 pParam->mcb = pParam->pThis->GetNextMethodContextHelper();
235 }
236 PAL_EXCEPT_FILTER(FilterSuperPMIExceptions_CaptureExceptionAndStop)
237 {
238 LogError("Method %d is of low integrity.", GetMethodContextIndex());
239 param.mcb = MethodContextBuffer(-1);
240 }
241 PAL_ENDTRY
242
243 return param.mcb;
244}
245
246MethodContextBuffer MethodContextReader::GetNextMethodContextHelper()
247{
248 // If we have an offset/increment combo
249 if (this->Offset > 0 && this->Increment > 0)
250 return GetNextMethodContextFromOffsetIncrement();
251
252 // If we have an index
253 if (this->hasIndex())
254 {
255 if (this->curIndexPos < this->IndexCount)
256 {
257 // If we are not done with all of them
258 return GetNextMethodContextFromIndexes();
259 }
260 else // We are done with all of them, return
261 return MethodContextBuffer();
262 }
263
264 // If we have a hash
265 if (this->Hash != nullptr)
266 return GetNextMethodContextFromHash();
267
268 // If we don't have any of these options return all MCs one by one
269 return this->ReadMethodContext(true);
270}
271
272// Read a method context buffer from the ContextCollection using Indexes
273MethodContextBuffer MethodContextReader::GetNextMethodContextFromIndexes()
274{
275 // Assert if we don't have an Index or we are done with all the indexes
276 Assert(this->hasIndex() && this->curIndexPos < this->IndexCount);
277
278 if (this->hasTOC())
279 {
280 // If we have an index & we have a TOC, we can just jump to that method!
281 return this->GetSpecificMethodContext(this->Indexes[this->curIndexPos++]);
282 }
283 else
284 {
285 // Find the current method (either #0, or the previous index)
286 int curMethod = this->curIndexPos ? this->Indexes[this->curIndexPos - 1] : 0;
287 // Get the next method
288 int nextMethod = this->Indexes[this->curIndexPos++];
289 // Skip over methods until we get to the right now
290 while (++curMethod < nextMethod)
291 {
292 // Skip a method context
293 MethodContextBuffer mcb = this->ReadMethodContext(true, true);
294 if (mcb.allDone() || mcb.Error())
295 return mcb;
296 }
297 }
298 return this->ReadMethodContext(true);
299}
300
301// Read a method context buffer from the ContextCollection using Hash
302MethodContextBuffer MethodContextReader::GetNextMethodContextFromHash()
303{
304 // Assert if we don't have a valid hash
305 Assert(this->Hash != nullptr);
306
307 if (this->hasTOC())
308 {
309 // We have a TOC so lets go through the TOCElements
310 // one-by-one till we find a matching hash
311 for (; curTOCIndex < (int)this->tocFile.GetTocCount(); curTOCIndex++)
312 {
313 if (_strnicmp(this->Hash, this->tocFile.GetElementPtr(curTOCIndex)->Hash, MD5_HASH_BUFFER_SIZE) == 0)
314 {
315 // We found a match, return this specific method
316 return this->GetSpecificMethodContext(this->tocFile.GetElementPtr(curTOCIndex++)->Number);
317 }
318 }
319
320 // No more matches in the TOC for our hash value
321 return MethodContextBuffer();
322 }
323 else
324 {
325 // Keep reading all MCs until we hit a match
326 // or we reach the end or hit an error
327 while (true)
328 {
329 // Read a method context
330 // we can't skip because we need to calculate hashes
331 MethodContextBuffer mcb = this->ReadMethodContext(true, false);
332 if (mcb.allDone() || mcb.Error())
333 return mcb;
334
335 char mcHash[MD5_HASH_BUFFER_SIZE];
336
337 // Create a temporary copy of mcb.buff plus ending 2-byte canary
338 // this will get freed up by MethodContext constructor
339 unsigned char* buff = new unsigned char[mcb.size + 2];
340 memcpy(buff, mcb.buff, mcb.size + 2);
341
342 MethodContext* mc;
343
344 if (!MethodContext::Initialize(-1, buff, mcb.size, &mc))
345 return MethodContextBuffer(-1);
346
347 mc->dumpMethodMD5HashToBuffer(mcHash, MD5_HASH_BUFFER_SIZE);
348 delete mc;
349
350 if (_strnicmp(this->Hash, mcHash, MD5_HASH_BUFFER_SIZE) == 0)
351 {
352 // We found a match, return this specific method
353 return mcb;
354 }
355 }
356 }
357
358 // We should never get here under normal conditions
359 AssertMsg(true, "Unexpected condition hit while reading input file.");
360 return MethodContextBuffer(-1);
361}
362
363// Read a method context buffer from the ContextCollection using offset/increment
364MethodContextBuffer MethodContextReader::GetNextMethodContextFromOffsetIncrement()
365{
366 // Assert if we don't have a valid increment/offset combo
367 Assert(this->Offset > 0 && this->Increment > 0);
368
369 int methodNumber = this->curMCIndex > 0 ? this->curMCIndex + this->Increment : this->Offset;
370
371 if (this->hasTOC())
372 {
373 // Check if we are within the TOC
374 if ((int)this->tocFile.GetTocCount() >= methodNumber)
375 {
376 // We have a TOC so we can request a specific method context
377 return this->GetSpecificMethodContext(methodNumber);
378 }
379 else
380 return MethodContextBuffer();
381 }
382 else
383 {
384 // Keep skipping MCs until we get to the one we need to return
385 while (this->curMCIndex + 1 < methodNumber)
386 {
387 // skip over a method
388 MethodContextBuffer mcb = this->ReadMethodContext(true, true);
389 if (mcb.allDone() || mcb.Error())
390 return mcb;
391 }
392 }
393 return this->ReadMethodContext(true);
394}
395
396bool MethodContextReader::hasIndex()
397{
398 return this->IndexCount > 0;
399}
400
401bool MethodContextReader::hasTOC()
402{
403 return this->tocFile.GetTocCount() > 0;
404}
405
406bool MethodContextReader::isValid()
407{
408 return this->fileHandle != INVALID_HANDLE_VALUE && this->mutex != INVALID_HANDLE_VALUE;
409}
410
411double MethodContextReader::PercentComplete()
412{
413 if (this->hasIndex() && this->hasTOC())
414 {
415 // Best estimate I can come up with...
416 return 100.0 * (double)this->curIndexPos / (double)this->IndexCount;
417 }
418 this->AcquireLock();
419 __int64 pos = 0;
420 SetFilePointerEx(this->fileHandle, *(PLARGE_INTEGER)&pos, (PLARGE_INTEGER)&pos, FILE_CURRENT);
421 this->ReleaseLock();
422 return 100.0 * (double)pos / (double)this->fileSize;
423}
424
425// Binary search to get this method number from the index
426// Returns -1 for not found, or -2 for not indexed
427// Interview question alert: hurray for CLR headers incompatibility with STL :-(
428// Note that TOC is 0 based and MC# are 1 based!
429__int64 MethodContextReader::GetOffset(unsigned int methodNumber)
430{
431 if (!this->hasTOC())
432 return -2;
433 size_t high = this->tocFile.GetTocCount() - 1;
434 size_t low = 0;
435 while (low <= high)
436 {
437 size_t pos = (high + low) / 2;
438 unsigned int num = this->tocFile.GetElementPtr(pos)->Number;
439 if (num == methodNumber)
440 return this->tocFile.GetElementPtr(pos)->Offset;
441 if (num > methodNumber)
442 high = pos - 1;
443 else
444 low = pos + 1;
445 }
446 return -1;
447}
448
449MethodContextBuffer MethodContextReader::GetSpecificMethodContext(unsigned int methodNumber)
450{
451 __int64 pos = this->GetOffset(methodNumber);
452 if (pos < 0)
453 {
454 return MethodContextBuffer(-3);
455 }
456
457 // Take the IO lock before we set the file pointer, so we can do this on multiple threads
458 if (!this->AcquireLock())
459 {
460 return MethodContextBuffer(-2);
461 }
462 if (SetFilePointerEx(this->fileHandle, *(PLARGE_INTEGER)&pos, (PLARGE_INTEGER)&pos, FILE_BEGIN))
463 {
464 // ReadMethodContext will release the lock, but we already acquired it
465 MethodContextBuffer mcb = this->ReadMethodContext(false);
466
467 // The curMCIndex value updated by ReadMethodContext() is incorrect
468 // since we are repositioning the file pointer we need to update it
469 curMCIndex = methodNumber;
470
471 return mcb;
472 }
473 else
474 {
475 // Don't forget to release the lock!
476 this->ReleaseLock();
477 return MethodContextBuffer(-4);
478 }
479}
480
481// Read the file with excluded methods hashes and save them.
482void MethodContextReader::ReadExcludedMethods(std::string mchFileName)
483{
484 excludedMethodsList = nullptr;
485
486 size_t suffix_offset = mchFileName.find_last_of('.');
487 if (suffix_offset == std::string::npos)
488 {
489 LogError("Failed to get file extension from %s", mchFileName.c_str());
490 return;
491 }
492 std::string suffix = mchFileName.substr(suffix_offset);
493 std::string excludeFileName = MethodContextReader::CheckForPairedFile(mchFileName, suffix.c_str(), ".exc");
494
495 if (excludeFileName.empty())
496 {
497 return;
498 }
499 HANDLE excludeFileHandle = OpenFile(excludeFileName.c_str());
500 if (excludeFileHandle != INVALID_HANDLE_VALUE)
501 {
502 __int64 excludeFileSizeLong;
503 GetFileSizeEx(excludeFileHandle, (PLARGE_INTEGER)&excludeFileSizeLong);
504 unsigned excludeFileSize = (unsigned)excludeFileSizeLong;
505
506 char* buffer = new char[excludeFileSize + 1];
507 DWORD bytesRead;
508 bool success = (ReadFile(excludeFileHandle, buffer, excludeFileSize, &bytesRead, NULL) == TRUE);
509 CloseHandle(excludeFileHandle);
510
511 if (!success || excludeFileSize != bytesRead)
512 {
513 LogError("Failed to read the exclude file.");
514 delete[] buffer;
515 return;
516 }
517
518 buffer[excludeFileSize] = 0;
519
520 int counter = 0;
521
522 char* curr = buffer;
523 while (*curr != 0)
524 {
525 while (isspace(*curr))
526 {
527 curr++;
528 }
529
530 std::string hash;
531 while (*curr != 0 && !isspace(*curr))
532 {
533 hash += *curr;
534 curr++;
535 }
536
537 if (hash.length() == MD5_HASH_BUFFER_SIZE - 1)
538 {
539 StringList* node = new StringList();
540 node->hash = hash;
541 node->next = excludedMethodsList;
542 excludedMethodsList = node;
543 counter++;
544 }
545 else
546 {
547 LogInfo("The exclude file contains wrong values: %s.", hash.c_str());
548 }
549 }
550 delete[] buffer;
551 LogInfo("Exclude file %s contains %d methods.", excludeFileName.c_str(), counter);
552 }
553}
554
555// Free memory used for excluded methods.
556void MethodContextReader::CleanExcludedMethods()
557{
558 while (excludedMethodsList != nullptr)
559 {
560 StringList* next = excludedMethodsList->next;
561 delete excludedMethodsList;
562 excludedMethodsList = next;
563 }
564}
565
566// Return should this method context be excluded from the replay or not.
567bool MethodContextReader::IsMethodExcluded(MethodContext* mc)
568{
569 if (excludedMethodsList != nullptr)
570 {
571 char md5HashBuf[MD5_HASH_BUFFER_SIZE] = {0};
572 mc->dumpMethodMD5HashToBuffer(md5HashBuf, MD5_HASH_BUFFER_SIZE);
573 for (StringList* node = excludedMethodsList; node != nullptr; node = node->next)
574 {
575 if (strcmp(node->hash.c_str(), md5HashBuf) == 0)
576 {
577 return true;
578 }
579 }
580 }
581 return false;
582}
583