1//
2// Copyright (c) Microsoft. All rights reserved.
3// Licensed under the MIT license. See LICENSE file in the project root for full license information.
4//
5
6#include "standardpch.h"
7#include "verbremovedup.h"
8#include "simpletimer.h"
9#include "lightweightmap.h"
10#include "methodcontext.h"
11#include "methodcontextiterator.h"
12
13// We use a hash to limit the number of comparisons we need to do.
14// The first level key to our hash map is ILCodeSize and the second
15// level map key is just an index and the value is an existing MC Hash.
16
17LightWeightMap<int, DenseLightWeightMap<char*>*>* inFile = nullptr;
18
19bool unique(MethodContext* mc)
20{
21 if (inFile == nullptr)
22 inFile = new LightWeightMap<int, DenseLightWeightMap<char*>*>();
23
24 CORINFO_METHOD_INFO newInfo;
25 unsigned newFlags = 0;
26 mc->repCompileMethod(&newInfo, &newFlags);
27
28 char* md5Buff = new char[MD5_HASH_BUFFER_SIZE];
29 mc->dumpMethodMD5HashToBuffer(md5Buff, MD5_HASH_BUFFER_SIZE);
30
31 if (inFile->GetIndex(newInfo.ILCodeSize) == -1)
32 inFile->Add(newInfo.ILCodeSize, new DenseLightWeightMap<char*>());
33
34 DenseLightWeightMap<char*>* ourRank = inFile->Get(newInfo.ILCodeSize);
35
36 for (int i = 0; i < (int)ourRank->GetCount(); i++)
37 {
38 char* md5Buff2 = ourRank->Get(i);
39
40 if (strncmp(md5Buff, md5Buff2, MD5_HASH_BUFFER_SIZE) == 0)
41 {
42 delete[] md5Buff;
43 return false;
44 }
45 }
46
47 ourRank->Append(md5Buff);
48 return true;
49}
50
51LightWeightMap<int, DenseLightWeightMap<MethodContext*>*>* inFileLegacy = nullptr;
52
53bool uniqueLegacy(MethodContext* mc)
54{
55 if (inFileLegacy == nullptr)
56 inFileLegacy = new LightWeightMap<int, DenseLightWeightMap<MethodContext*>*>();
57
58 CORINFO_METHOD_INFO newInfo;
59 unsigned newFlags = 0;
60 mc->repCompileMethod(&newInfo, &newFlags);
61
62 if (inFileLegacy->GetIndex(newInfo.ILCodeSize) == -1)
63 inFileLegacy->Add(newInfo.ILCodeSize, new DenseLightWeightMap<MethodContext*>());
64
65 DenseLightWeightMap<MethodContext*>* ourRank = inFileLegacy->Get(newInfo.ILCodeSize);
66
67 for (int i = 0; i < (int)ourRank->GetCount(); i++)
68 {
69 MethodContext* scratch = ourRank->Get(i);
70 if (mc->Equal(scratch))
71 {
72 return false;
73 }
74 }
75
76 // We store the MethodContext in our map.
77 ourRank->Append(mc);
78 return true;
79}
80
81int verbRemoveDup::DoWork(const char* nameOfInput, const char* nameOfOutput, bool stripCR, bool legacyCompare)
82{
83 LogVerbose("Removing duplicates from '%s', writing to '%s'", nameOfInput, nameOfOutput);
84
85 MethodContextIterator mci(true);
86 if (!mci.Initialize(nameOfInput))
87 return -1;
88
89 int savedCount = 0;
90
91 HANDLE hFileOut = CreateFileA(nameOfOutput, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS,
92 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, NULL);
93 if (hFileOut == INVALID_HANDLE_VALUE)
94 {
95 LogError("Failed to open output '%s'. GetLastError()=%u", nameOfOutput, GetLastError());
96 return -1;
97 }
98
99 while (mci.MoveNext())
100 {
101 MethodContext* mc = mci.CurrentTakeOwnership();
102 if (stripCR)
103 {
104 delete mc->cr;
105 mc->cr = new CompileResult();
106 }
107 if (legacyCompare)
108 {
109 if (uniqueLegacy(mc))
110 {
111 mc->saveToFile(hFileOut);
112 savedCount++;
113
114 // In this case, for the legacy comparer, it has placed the 'mc' in the 'inFileLegacy' table, so we
115 // can't delete it.
116 }
117 else
118 {
119 delete mc; // we no longer need this
120 }
121 }
122 else
123 {
124 if (unique(mc))
125 {
126 mc->saveToFile(hFileOut);
127 savedCount++;
128 }
129 delete mc; // we no longer need this
130 }
131 }
132
133 // We're leaking 'inFile' or 'inFileLegacy', but the process is going away, so it shouldn't matter.
134
135 if (CloseHandle(hFileOut) == 0)
136 {
137 LogError("2nd CloseHandle failed. GetLastError()=%u", GetLastError());
138 return -1;
139 }
140
141 LogInfo("Loaded %d, Saved %d", mci.MethodContextNumber(), savedCount);
142
143 if (!mci.Destroy())
144 return -1;
145
146 return 0;
147}
148