1//===--- SemanticHighlighting.cpp - ------------------------- ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SemanticHighlighting.h"
10#include "Config.h"
11#include "FindTarget.h"
12#include "HeuristicResolver.h"
13#include "ParsedAST.h"
14#include "Protocol.h"
15#include "SourceCode.h"
16#include "support/Logger.h"
17#include "clang/AST/ASTContext.h"
18#include "clang/AST/Decl.h"
19#include "clang/AST/DeclCXX.h"
20#include "clang/AST/DeclObjC.h"
21#include "clang/AST/DeclTemplate.h"
22#include "clang/AST/DeclarationName.h"
23#include "clang/AST/ExprCXX.h"
24#include "clang/AST/RecursiveASTVisitor.h"
25#include "clang/AST/Type.h"
26#include "clang/AST/TypeLoc.h"
27#include "clang/Basic/LangOptions.h"
28#include "clang/Basic/SourceLocation.h"
29#include "clang/Basic/SourceManager.h"
30#include "clang/Tooling/Syntax/Tokens.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/Error.h"
35#include <algorithm>
36#include <optional>
37
38namespace clang {
39namespace clangd {
40namespace {
41
42/// Get the last Position on a given line.
43llvm::Expected<Position> endOfLine(llvm::StringRef Code, int Line) {
44 auto StartOfLine = positionToOffset(Code, Position{Line, 0});
45 if (!StartOfLine)
46 return StartOfLine.takeError();
47 StringRef LineText = Code.drop_front(*StartOfLine).take_until([](char C) {
48 return C == '\n';
49 });
50 return Position{Line, static_cast<int>(lspLength(LineText))};
51}
52
53/// Some names are not written in the source code and cannot be highlighted,
54/// e.g. anonymous classes. This function detects those cases.
55bool canHighlightName(DeclarationName Name) {
56 switch (Name.getNameKind()) {
57 case DeclarationName::Identifier: {
58 auto *II = Name.getAsIdentifierInfo();
59 return II && !II->getName().empty();
60 }
61 case DeclarationName::CXXConstructorName:
62 case DeclarationName::CXXDestructorName:
63 return true;
64 case DeclarationName::ObjCZeroArgSelector:
65 case DeclarationName::ObjCOneArgSelector:
66 case DeclarationName::ObjCMultiArgSelector:
67 // Multi-arg selectors need special handling, and we handle 0/1 arg
68 // selectors there too.
69 return false;
70 case DeclarationName::CXXConversionFunctionName:
71 case DeclarationName::CXXOperatorName:
72 case DeclarationName::CXXDeductionGuideName:
73 case DeclarationName::CXXLiteralOperatorName:
74 case DeclarationName::CXXUsingDirective:
75 return false;
76 }
77 llvm_unreachable("invalid name kind");
78}
79
80bool isUniqueDefinition(const NamedDecl *Decl) {
81 if (auto *Func = dyn_cast<FunctionDecl>(Decl))
82 return Func->isThisDeclarationADefinition();
83 if (auto *Klass = dyn_cast<CXXRecordDecl>(Decl))
84 return Klass->isThisDeclarationADefinition();
85 if (auto *Iface = dyn_cast<ObjCInterfaceDecl>(Decl))
86 return Iface->isThisDeclarationADefinition();
87 if (auto *Proto = dyn_cast<ObjCProtocolDecl>(Decl))
88 return Proto->isThisDeclarationADefinition();
89 if (auto *Var = dyn_cast<VarDecl>(Decl))
90 return Var->isThisDeclarationADefinition();
91 return isa<TemplateTypeParmDecl>(Decl) ||
92 isa<NonTypeTemplateParmDecl>(Decl) ||
93 isa<TemplateTemplateParmDecl>(Decl) || isa<ObjCCategoryDecl>(Decl) ||
94 isa<ObjCImplDecl>(Decl);
95}
96
97std::optional<HighlightingKind> kindForType(const Type *TP,
98 const HeuristicResolver *Resolver);
99std::optional<HighlightingKind> kindForDecl(const NamedDecl *D,
100 const HeuristicResolver *Resolver) {
101 if (auto *USD = dyn_cast<UsingShadowDecl>(D)) {
102 if (auto *Target = USD->getTargetDecl())
103 D = Target;
104 }
105 if (auto *TD = dyn_cast<TemplateDecl>(D)) {
106 if (auto *Templated = TD->getTemplatedDecl())
107 D = Templated;
108 }
109 if (auto *TD = dyn_cast<TypedefNameDecl>(D)) {
110 // We try to highlight typedefs as their underlying type.
111 if (auto K =
112 kindForType(TD->getUnderlyingType().getTypePtrOrNull(), Resolver))
113 return K;
114 // And fallback to a generic kind if this fails.
115 return HighlightingKind::Typedef;
116 }
117 // We highlight class decls, constructor decls and destructor decls as
118 // `Class` type. The destructor decls are handled in `VisitTagTypeLoc` (we
119 // will visit a TypeLoc where the underlying Type is a CXXRecordDecl).
120 if (auto *RD = llvm::dyn_cast<RecordDecl>(D)) {
121 // We don't want to highlight lambdas like classes.
122 if (RD->isLambda())
123 return std::nullopt;
124 return HighlightingKind::Class;
125 }
126 if (isa<ClassTemplateDecl, RecordDecl, CXXConstructorDecl, ObjCInterfaceDecl,
127 ObjCImplementationDecl>(D))
128 return HighlightingKind::Class;
129 if (isa<ObjCProtocolDecl>(D))
130 return HighlightingKind::Interface;
131 if (isa<ObjCCategoryDecl, ObjCCategoryImplDecl>(D))
132 return HighlightingKind::Namespace;
133 if (auto *MD = dyn_cast<CXXMethodDecl>(D))
134 return MD->isStatic() ? HighlightingKind::StaticMethod
135 : HighlightingKind::Method;
136 if (auto *OMD = dyn_cast<ObjCMethodDecl>(D))
137 return OMD->isClassMethod() ? HighlightingKind::StaticMethod
138 : HighlightingKind::Method;
139 if (isa<FieldDecl, ObjCPropertyDecl>(D))
140 return HighlightingKind::Field;
141 if (isa<EnumDecl>(D))
142 return HighlightingKind::Enum;
143 if (isa<EnumConstantDecl>(D))
144 return HighlightingKind::EnumConstant;
145 if (isa<ParmVarDecl>(D))
146 return HighlightingKind::Parameter;
147 if (auto *VD = dyn_cast<VarDecl>(D)) {
148 if (isa<ImplicitParamDecl>(VD)) // e.g. ObjC Self
149 return std::nullopt;
150 return VD->isStaticDataMember()
151 ? HighlightingKind::StaticField
152 : VD->isLocalVarDecl() ? HighlightingKind::LocalVariable
153 : HighlightingKind::Variable;
154 }
155 if (const auto *BD = dyn_cast<BindingDecl>(D))
156 return BD->getDeclContext()->isFunctionOrMethod()
157 ? HighlightingKind::LocalVariable
158 : HighlightingKind::Variable;
159 if (isa<FunctionDecl>(D))
160 return HighlightingKind::Function;
161 if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||
162 isa<UsingDirectiveDecl>(D))
163 return HighlightingKind::Namespace;
164 if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
165 isa<NonTypeTemplateParmDecl>(D))
166 return HighlightingKind::TemplateParameter;
167 if (isa<ConceptDecl>(D))
168 return HighlightingKind::Concept;
169 if (isa<LabelDecl>(D))
170 return HighlightingKind::Label;
171 if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(D)) {
172 auto Targets = Resolver->resolveUsingValueDecl(UUVD);
173 if (!Targets.empty() && Targets[0] != UUVD) {
174 return kindForDecl(Targets[0], Resolver);
175 }
176 return HighlightingKind::Unknown;
177 }
178 return std::nullopt;
179}
180std::optional<HighlightingKind> kindForType(const Type *TP,
181 const HeuristicResolver *Resolver) {
182 if (!TP)
183 return std::nullopt;
184 if (TP->isBuiltinType()) // Builtins are special, they do not have decls.
185 return HighlightingKind::Primitive;
186 if (auto *TD = dyn_cast<TemplateTypeParmType>(TP))
187 return kindForDecl(TD->getDecl(), Resolver);
188 if (isa<ObjCObjectPointerType>(TP))
189 return HighlightingKind::Class;
190 if (auto *TD = TP->getAsTagDecl())
191 return kindForDecl(TD, Resolver);
192 return std::nullopt;
193}
194
195// Whether T is const in a loose sense - is a variable with this type readonly?
196bool isConst(QualType T) {
197 if (T.isNull())
198 return false;
199 T = T.getNonReferenceType();
200 if (T.isConstQualified())
201 return true;
202 if (const auto *AT = T->getAsArrayTypeUnsafe())
203 return isConst(AT->getElementType());
204 if (isConst(T->getPointeeType()))
205 return true;
206 return false;
207}
208
209// Whether D is const in a loose sense (should it be highlighted as such?)
210// FIXME: This is separate from whether *a particular usage* can mutate D.
211// We may want V in V.size() to be readonly even if V is mutable.
212bool isConst(const Decl *D) {
213 if (llvm::isa<EnumConstantDecl>(D) || llvm::isa<NonTypeTemplateParmDecl>(D))
214 return true;
215 if (llvm::isa<FieldDecl>(D) || llvm::isa<VarDecl>(D) ||
216 llvm::isa<MSPropertyDecl>(D) || llvm::isa<BindingDecl>(D)) {
217 if (isConst(llvm::cast<ValueDecl>(D)->getType()))
218 return true;
219 }
220 if (const auto *OCPD = llvm::dyn_cast<ObjCPropertyDecl>(D)) {
221 if (OCPD->isReadOnly())
222 return true;
223 }
224 if (const auto *MPD = llvm::dyn_cast<MSPropertyDecl>(D)) {
225 if (!MPD->hasSetter())
226 return true;
227 }
228 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {
229 if (CMD->isConst())
230 return true;
231 }
232 return false;
233}
234
235// "Static" means many things in C++, only some get the "static" modifier.
236//
237// Meanings that do:
238// - Members associated with the class rather than the instance.
239// This is what 'static' most often means across languages.
240// - static local variables
241// These are similarly "detached from their context" by the static keyword.
242// In practice, these are rarely used inside classes, reducing confusion.
243//
244// Meanings that don't:
245// - Namespace-scoped variables, which have static storage class.
246// This is implicit, so the keyword "static" isn't so strongly associated.
247// If we want a modifier for these, "global scope" is probably the concept.
248// - Namespace-scoped variables/functions explicitly marked "static".
249// There the keyword changes *linkage* , which is a totally different concept.
250// If we want to model this, "file scope" would be a nice modifier.
251//
252// This is confusing, and maybe we should use another name, but because "static"
253// is a standard LSP modifier, having one with that name has advantages.
254bool isStatic(const Decl *D) {
255 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
256 return CMD->isStatic();
257 if (const VarDecl *VD = llvm::dyn_cast<VarDecl>(D))
258 return VD->isStaticDataMember() || VD->isStaticLocal();
259 if (const auto *OPD = llvm::dyn_cast<ObjCPropertyDecl>(D))
260 return OPD->isClassProperty();
261 if (const auto *OMD = llvm::dyn_cast<ObjCMethodDecl>(D))
262 return OMD->isClassMethod();
263 return false;
264}
265
266bool isAbstract(const Decl *D) {
267 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
268 return CMD->isPure();
269 if (const auto *CRD = llvm::dyn_cast<CXXRecordDecl>(D))
270 return CRD->hasDefinition() && CRD->isAbstract();
271 return false;
272}
273
274bool isVirtual(const Decl *D) {
275 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
276 return CMD->isVirtual();
277 return false;
278}
279
280bool isDependent(const Decl *D) {
281 if (isa<UnresolvedUsingValueDecl>(D))
282 return true;
283 return false;
284}
285
286/// Returns true if `Decl` is considered to be from a default/system library.
287/// This currently checks the systemness of the file by include type, although
288/// different heuristics may be used in the future (e.g. sysroot paths).
289bool isDefaultLibrary(const Decl *D) {
290 SourceLocation Loc = D->getLocation();
291 if (!Loc.isValid())
292 return false;
293 return D->getASTContext().getSourceManager().isInSystemHeader(Loc);
294}
295
296bool isDefaultLibrary(const Type *T) {
297 if (!T)
298 return false;
299 const Type *Underlying = T->getPointeeOrArrayElementType();
300 if (Underlying->isBuiltinType())
301 return true;
302 if (auto *TD = dyn_cast<TemplateTypeParmType>(Underlying))
303 return isDefaultLibrary(TD->getDecl());
304 if (auto *TD = Underlying->getAsTagDecl())
305 return isDefaultLibrary(TD);
306 return false;
307}
308
309// For a macro usage `DUMP(foo)`, we want:
310// - DUMP --> "macro"
311// - foo --> "variable".
312SourceLocation getHighlightableSpellingToken(SourceLocation L,
313 const SourceManager &SM) {
314 if (L.isFileID())
315 return SM.isWrittenInMainFile(L) ? L : SourceLocation{};
316 // Tokens expanded from the macro body contribute no highlightings.
317 if (!SM.isMacroArgExpansion(L))
318 return {};
319 // Tokens expanded from macro args are potentially highlightable.
320 return getHighlightableSpellingToken(SM.getImmediateSpellingLoc(L), SM);
321}
322
323unsigned evaluateHighlightPriority(const HighlightingToken &Tok) {
324 enum HighlightPriority { Dependent = 0, Resolved = 1 };
325 return (Tok.Modifiers & (1 << uint32_t(HighlightingModifier::DependentName)))
326 ? Dependent
327 : Resolved;
328}
329
330// Sometimes we get multiple tokens at the same location:
331//
332// - findExplicitReferences() returns a heuristic result for a dependent name
333// (e.g. Method) and CollectExtraHighlighting returning a fallback dependent
334// highlighting (e.g. Unknown+Dependent).
335// - macro arguments are expanded multiple times and have different roles
336// - broken code recovery produces several AST nodes at the same location
337//
338// We should either resolve these to a single token, or drop them all.
339// Our heuristics are:
340//
341// - token kinds that come with "dependent-name" modifiers are less reliable
342// (these tend to be vague, like Type or Unknown)
343// - if we have multiple equally reliable kinds, drop token rather than guess
344// - take the union of modifiers from all tokens
345//
346// In particular, heuristically resolved dependent names get their heuristic
347// kind, plus the dependent modifier.
348std::optional<HighlightingToken> resolveConflict(const HighlightingToken &A,
349 const HighlightingToken &B) {
350 unsigned Priority1 = evaluateHighlightPriority(A);
351 unsigned Priority2 = evaluateHighlightPriority(B);
352 if (Priority1 == Priority2 && A.Kind != B.Kind)
353 return std::nullopt;
354 auto Result = Priority1 > Priority2 ? A : B;
355 Result.Modifiers = A.Modifiers | B.Modifiers;
356 return Result;
357}
358std::optional<HighlightingToken>
359resolveConflict(ArrayRef<HighlightingToken> Tokens) {
360 if (Tokens.size() == 1)
361 return Tokens[0];
362
363 assert(Tokens.size() >= 2);
364 std::optional<HighlightingToken> Winner =
365 resolveConflict(Tokens[0], Tokens[1]);
366 for (size_t I = 2; Winner && I < Tokens.size(); ++I)
367 Winner = resolveConflict(*Winner, Tokens[I]);
368 return Winner;
369}
370
371/// Filter to remove particular kinds of highlighting tokens and modifiers from
372/// the output.
373class HighlightingFilter {
374public:
375 HighlightingFilter() {
376 for (auto &Active : ActiveKindLookup)
377 Active = true;
378
379 ActiveModifiersMask = ~0;
380 }
381
382 void disableKind(HighlightingKind Kind) {
383 ActiveKindLookup[static_cast<size_t>(Kind)] = false;
384 }
385
386 void disableModifier(HighlightingModifier Modifier) {
387 ActiveModifiersMask &= ~(1 << static_cast<uint32_t>(Modifier));
388 }
389
390 bool isHighlightKindActive(HighlightingKind Kind) const {
391 return ActiveKindLookup[static_cast<size_t>(Kind)];
392 }
393
394 uint32_t maskModifiers(uint32_t Modifiers) const {
395 return Modifiers & ActiveModifiersMask;
396 }
397
398 static HighlightingFilter fromCurrentConfig() {
399 const Config &C = Config::current();
400 HighlightingFilter Filter;
401 for (const auto &Kind : C.SemanticTokens.DisabledKinds)
402 if (auto K = highlightingKindFromString(Kind))
403 Filter.disableKind(*K);
404 for (const auto &Modifier : C.SemanticTokens.DisabledModifiers)
405 if (auto M = highlightingModifierFromString(Modifier))
406 Filter.disableModifier(*M);
407
408 return Filter;
409 }
410
411private:
412 bool ActiveKindLookup[static_cast<size_t>(HighlightingKind::LastKind) + 1];
413 uint32_t ActiveModifiersMask;
414};
415
416/// Consumes source locations and maps them to text ranges for highlightings.
417class HighlightingsBuilder {
418public:
419 HighlightingsBuilder(const ParsedAST &AST, const HighlightingFilter &Filter)
420 : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
421 LangOpts(AST.getLangOpts()), Filter(Filter) {}
422
423 HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
424 auto Range = getRangeForSourceLocation(Loc);
425 if (!Range)
426 return InvalidHighlightingToken;
427
428 return addToken(*Range, Kind);
429 }
430
431 // Most of this function works around
432 // https://github.com/clangd/clangd/issues/871.
433 void addAngleBracketTokens(SourceLocation LLoc, SourceLocation RLoc) {
434 if (!LLoc.isValid() || !RLoc.isValid())
435 return;
436
437 auto LRange = getRangeForSourceLocation(LLoc);
438 if (!LRange)
439 return;
440
441 // RLoc might be pointing at a virtual buffer when it's part of a `>>`
442 // token.
443 RLoc = SourceMgr.getFileLoc(RLoc);
444 // Make sure token is part of the main file.
445 RLoc = getHighlightableSpellingToken(RLoc, SourceMgr);
446 if (!RLoc.isValid())
447 return;
448
449 const auto *RTok = TB.spelledTokenAt(RLoc);
450 // Handle `>>`. RLoc is always pointing at the right location, just change
451 // the end to be offset by 1.
452 // We'll either point at the beginning of `>>`, hence get a proper spelled
453 // or point in the middle of `>>` hence get no spelled tok.
454 if (!RTok || RTok->kind() == tok::greatergreater) {
455 Position Begin = sourceLocToPosition(SourceMgr, RLoc);
456 Position End = sourceLocToPosition(SourceMgr, RLoc.getLocWithOffset(1));
457 addToken(*LRange, HighlightingKind::Bracket);
458 addToken({Begin, End}, HighlightingKind::Bracket);
459 return;
460 }
461
462 // Easy case, we have the `>` token directly available.
463 if (RTok->kind() == tok::greater) {
464 if (auto RRange = getRangeForSourceLocation(RLoc)) {
465 addToken(*LRange, HighlightingKind::Bracket);
466 addToken(*RRange, HighlightingKind::Bracket);
467 }
468 return;
469 }
470 }
471
472 HighlightingToken &addToken(Range R, HighlightingKind Kind) {
473 if (!Filter.isHighlightKindActive(Kind))
474 return InvalidHighlightingToken;
475
476 HighlightingToken HT;
477 HT.R = std::move(R);
478 HT.Kind = Kind;
479 Tokens.push_back(std::move(HT));
480 return Tokens.back();
481 }
482
483 void addExtraModifier(SourceLocation Loc, HighlightingModifier Modifier) {
484 if (auto Range = getRangeForSourceLocation(Loc))
485 ExtraModifiers[*Range].push_back(Modifier);
486 }
487
488 std::vector<HighlightingToken> collect(ParsedAST &AST) && {
489 // Initializer lists can give duplicates of tokens, therefore all tokens
490 // must be deduplicated.
491 llvm::sort(Tokens);
492 auto Last = std::unique(Tokens.begin(), Tokens.end());
493 Tokens.erase(Last, Tokens.end());
494
495 // Macros can give tokens that have the same source range but conflicting
496 // kinds. In this case all tokens sharing this source range should be
497 // removed.
498 std::vector<HighlightingToken> NonConflicting;
499 NonConflicting.reserve(Tokens.size());
500 for (ArrayRef<HighlightingToken> TokRef = Tokens; !TokRef.empty();) {
501 ArrayRef<HighlightingToken> Conflicting =
502 TokRef.take_while([&](const HighlightingToken &T) {
503 // TokRef is guaranteed at least one element here because otherwise
504 // this predicate would never fire.
505 return T.R == TokRef.front().R;
506 });
507 if (auto Resolved = resolveConflict(Conflicting)) {
508 // Apply extra collected highlighting modifiers
509 auto Modifiers = ExtraModifiers.find(Resolved->R);
510 if (Modifiers != ExtraModifiers.end()) {
511 for (HighlightingModifier Mod : Modifiers->second) {
512 Resolved->addModifier(Mod);
513 }
514 }
515
516 Resolved->Modifiers = Filter.maskModifiers(Resolved->Modifiers);
517 NonConflicting.push_back(*Resolved);
518 }
519 // TokRef[Conflicting.size()] is the next token with a different range (or
520 // the end of the Tokens).
521 TokRef = TokRef.drop_front(Conflicting.size());
522 }
523
524 if (!Filter.isHighlightKindActive(HighlightingKind::InactiveCode))
525 return NonConflicting;
526
527 const auto &SM = AST.getSourceManager();
528 StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
529
530 // Merge token stream with "inactive line" markers.
531 std::vector<HighlightingToken> WithInactiveLines;
532 auto SortedInactiveRegions = getInactiveRegions(AST);
533 llvm::sort(SortedInactiveRegions);
534 auto It = NonConflicting.begin();
535 for (const Range &R : SortedInactiveRegions) {
536 // Create one token for each line in the inactive range, so it works
537 // with line-based diffing.
538 assert(R.start.line <= R.end.line);
539 for (int Line = R.start.line; Line <= R.end.line; ++Line) {
540 // Copy tokens before the inactive line
541 for (; It != NonConflicting.end() && It->R.start.line < Line; ++It)
542 WithInactiveLines.push_back(std::move(*It));
543 // Add a token for the inactive line itself.
544 auto EndOfLine = endOfLine(MainCode, Line);
545 if (EndOfLine) {
546 HighlightingToken HT;
547 WithInactiveLines.emplace_back();
548 WithInactiveLines.back().Kind = HighlightingKind::InactiveCode;
549 WithInactiveLines.back().R.start.line = Line;
550 WithInactiveLines.back().R.end = *EndOfLine;
551 } else {
552 elog("Failed to determine end of line: {0}", EndOfLine.takeError());
553 }
554
555 // Skip any other tokens on the inactive line. e.g.
556 // `#ifndef Foo` is considered as part of an inactive region when Foo is
557 // defined, and there is a Foo macro token.
558 // FIXME: we should reduce the scope of the inactive region to not
559 // include the directive itself.
560 while (It != NonConflicting.end() && It->R.start.line == Line)
561 ++It;
562 }
563 }
564 // Copy tokens after the last inactive line
565 for (; It != NonConflicting.end(); ++It)
566 WithInactiveLines.push_back(std::move(*It));
567 return WithInactiveLines;
568 }
569
570 const HeuristicResolver *getResolver() const { return Resolver; }
571
572private:
573 std::optional<Range> getRangeForSourceLocation(SourceLocation Loc) {
574 Loc = getHighlightableSpellingToken(Loc, SourceMgr);
575 if (Loc.isInvalid())
576 return std::nullopt;
577 // We might have offsets in the main file that don't correspond to any
578 // spelled tokens.
579 const auto *Tok = TB.spelledTokenAt(Loc);
580 if (!Tok)
581 return std::nullopt;
582 return halfOpenToRange(SourceMgr,
583 Tok->range(SourceMgr).toCharRange(SourceMgr));
584 }
585
586 const syntax::TokenBuffer &TB;
587 const SourceManager &SourceMgr;
588 const LangOptions &LangOpts;
589 HighlightingFilter Filter;
590 std::vector<HighlightingToken> Tokens;
591 std::map<Range, llvm::SmallVector<HighlightingModifier, 1>> ExtraModifiers;
592 const HeuristicResolver *Resolver = nullptr;
593 // returned from addToken(InvalidLoc)
594 HighlightingToken InvalidHighlightingToken;
595};
596
597std::optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {
598 const DeclContext *DC = D->getDeclContext();
599 // Injected "Foo" within the class "Foo" has file scope, not class scope.
600 if (auto *R = dyn_cast_or_null<RecordDecl>(D))
601 if (R->isInjectedClassName())
602 DC = DC->getParent();
603 // Lambda captures are considered function scope, not class scope.
604 if (llvm::isa<FieldDecl>(D))
605 if (const auto *RD = llvm::dyn_cast<RecordDecl>(DC))
606 if (RD->isLambda())
607 return HighlightingModifier::FunctionScope;
608 // Walk up the DeclContext hierarchy until we find something interesting.
609 for (; !DC->isFileContext(); DC = DC->getParent()) {
610 if (DC->isFunctionOrMethod())
611 return HighlightingModifier::FunctionScope;
612 if (DC->isRecord())
613 return HighlightingModifier::ClassScope;
614 }
615 // Some template parameters (e.g. those for variable templates) don't have
616 // meaningful DeclContexts. That doesn't mean they're global!
617 if (DC->isTranslationUnit() && D->isTemplateParameter())
618 return std::nullopt;
619 // ExternalLinkage threshold could be tweaked, e.g. module-visible as global.
620 if (D->getLinkageInternal() < ExternalLinkage)
621 return HighlightingModifier::FileScope;
622 return HighlightingModifier::GlobalScope;
623}
624
625std::optional<HighlightingModifier> scopeModifier(const Type *T) {
626 if (!T)
627 return std::nullopt;
628 if (T->isBuiltinType())
629 return HighlightingModifier::GlobalScope;
630 if (auto *TD = dyn_cast<TemplateTypeParmType>(T))
631 return scopeModifier(TD->getDecl());
632 if (auto *TD = T->getAsTagDecl())
633 return scopeModifier(TD);
634 return std::nullopt;
635}
636
637/// Produces highlightings, which are not captured by findExplicitReferences,
638/// e.g. highlights dependent names and 'auto' as the underlying type.
639class CollectExtraHighlightings
640 : public RecursiveASTVisitor<CollectExtraHighlightings> {
641 using Base = RecursiveASTVisitor<CollectExtraHighlightings>;
642
643public:
644 CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}
645
646 bool VisitCXXConstructExpr(CXXConstructExpr *E) {
647 highlightMutableReferenceArguments(E->getConstructor(),
648 {E->getArgs(), E->getNumArgs()});
649
650 return true;
651 }
652
653 bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {
654 if (Init->isMemberInitializer())
655 if (auto *Member = Init->getMember())
656 highlightMutableReferenceArgument(Member->getType(), Init->getInit());
657 return Base::TraverseConstructorInitializer(Init);
658 }
659
660 bool TraverseTypeConstraint(const TypeConstraint *C) {
661 if (auto *Args = C->getTemplateArgsAsWritten())
662 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
663 return Base::TraverseTypeConstraint(C);
664 }
665
666 bool VisitPredefinedExpr(PredefinedExpr *E) {
667 H.addToken(E->getLocation(), HighlightingKind::LocalVariable)
668 .addModifier(HighlightingModifier::Static)
669 .addModifier(HighlightingModifier::Readonly)
670 .addModifier(HighlightingModifier::FunctionScope);
671 return true;
672 }
673
674 bool VisitConceptSpecializationExpr(ConceptSpecializationExpr *E) {
675 if (auto *Args = E->getTemplateArgsAsWritten())
676 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
677 return true;
678 }
679
680 bool VisitTemplateDecl(TemplateDecl *D) {
681 if (auto *TPL = D->getTemplateParameters())
682 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
683 return true;
684 }
685
686 bool VisitTagDecl(TagDecl *D) {
687 for (unsigned i = 0; i < D->getNumTemplateParameterLists(); ++i) {
688 if (auto *TPL = D->getTemplateParameterList(i))
689 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
690 }
691 return true;
692 }
693
694 bool VisitClassTemplatePartialSpecializationDecl(
695 ClassTemplatePartialSpecializationDecl *D) {
696 if (auto *TPL = D->getTemplateParameters())
697 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
698 if (auto *Args = D->getTemplateArgsAsWritten())
699 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
700 return true;
701 }
702
703 bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *D) {
704 if (auto *Args = D->getTemplateArgsInfo())
705 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
706 return true;
707 }
708
709 bool VisitVarTemplatePartialSpecializationDecl(
710 VarTemplatePartialSpecializationDecl *D) {
711 if (auto *TPL = D->getTemplateParameters())
712 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
713 if (auto *Args = D->getTemplateArgsAsWritten())
714 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
715 return true;
716 }
717
718 bool VisitClassScopeFunctionSpecializationDecl(
719 ClassScopeFunctionSpecializationDecl *D) {
720 if (auto *Args = D->getTemplateArgsAsWritten())
721 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
722 return true;
723 }
724
725 bool VisitDeclRefExpr(DeclRefExpr *E) {
726 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
727 return true;
728 }
729 bool VisitMemberExpr(MemberExpr *E) {
730 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
731 return true;
732 }
733
734 bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc L) {
735 H.addAngleBracketTokens(L.getLAngleLoc(), L.getRAngleLoc());
736 return true;
737 }
738
739 bool VisitAutoTypeLoc(AutoTypeLoc L) {
740 if (L.isConstrained()) {
741 H.addAngleBracketTokens(L.getLAngleLoc(), L.getRAngleLoc());
742 H.addToken(L.getConceptNameInfo().getLoc(), HighlightingKind::Concept);
743 }
744 return true;
745 }
746
747 bool VisitFunctionDecl(FunctionDecl *D) {
748 if (D->isOverloadedOperator()) {
749 const auto AddOpDeclToken = [&](SourceLocation Loc) {
750 auto &Token = H.addToken(Loc, HighlightingKind::Operator)
751 .addModifier(HighlightingModifier::Declaration);
752 if (D->isThisDeclarationADefinition())
753 Token.addModifier(HighlightingModifier::Definition);
754 };
755 const auto Range = D->getNameInfo().getCXXOperatorNameRange();
756 AddOpDeclToken(Range.getBegin());
757 const auto Kind = D->getOverloadedOperator();
758 if (Kind == OO_Call || Kind == OO_Subscript)
759 AddOpDeclToken(Range.getEnd());
760 }
761 if (auto *Args = D->getTemplateSpecializationArgsAsWritten())
762 H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
763 if (auto *I = D->getDependentSpecializationInfo())
764 H.addAngleBracketTokens(I->getLAngleLoc(), I->getRAngleLoc());
765 return true;
766 }
767
768 bool VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
769 const auto AddOpToken = [&](SourceLocation Loc) {
770 H.addToken(Loc, HighlightingKind::Operator)
771 .addModifier(HighlightingModifier::UserDefined);
772 };
773 AddOpToken(E->getOperatorLoc());
774 const auto Kind = E->getOperator();
775 if (Kind == OO_Call || Kind == OO_Subscript) {
776 if (auto *Callee = E->getCallee())
777 AddOpToken(Callee->getBeginLoc());
778 }
779 return true;
780 }
781
782 bool VisitUnaryOperator(UnaryOperator *Op) {
783 auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);
784 if (Op->getSubExpr()->isTypeDependent())
785 Token.addModifier(HighlightingModifier::UserDefined);
786 return true;
787 }
788
789 bool VisitBinaryOperator(BinaryOperator *Op) {
790 auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);
791 if (Op->getLHS()->isTypeDependent() || Op->getRHS()->isTypeDependent())
792 Token.addModifier(HighlightingModifier::UserDefined);
793 return true;
794 }
795
796 bool VisitConditionalOperator(ConditionalOperator *Op) {
797 H.addToken(Op->getQuestionLoc(), HighlightingKind::Operator);
798 H.addToken(Op->getColonLoc(), HighlightingKind::Operator);
799 return true;
800 }
801
802 bool VisitCXXNewExpr(CXXNewExpr *E) {
803 auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);
804 if (isa_and_present<CXXMethodDecl>(E->getOperatorNew()))
805 Token.addModifier(HighlightingModifier::UserDefined);
806 return true;
807 }
808
809 bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
810 auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);
811 if (isa_and_present<CXXMethodDecl>(E->getOperatorDelete()))
812 Token.addModifier(HighlightingModifier::UserDefined);
813 return true;
814 }
815
816 bool VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
817 const auto &B = E->getAngleBrackets();
818 H.addAngleBracketTokens(B.getBegin(), B.getEnd());
819 return true;
820 }
821
822 bool VisitCallExpr(CallExpr *E) {
823 // Highlighting parameters passed by non-const reference does not really
824 // make sense for literals...
825 if (isa<UserDefinedLiteral>(E))
826 return true;
827
828 // FIXME: consider highlighting parameters of some other overloaded
829 // operators as well
830 llvm::ArrayRef<const Expr *> Args = {E->getArgs(), E->getNumArgs()};
831 if (auto *CallOp = dyn_cast<CXXOperatorCallExpr>(E)) {
832 switch (CallOp->getOperator()) {
833 case OO_Call:
834 case OO_Subscript:
835 Args = Args.drop_front(); // Drop object parameter
836 break;
837 default:
838 return true;
839 }
840 }
841
842 highlightMutableReferenceArguments(
843 dyn_cast_or_null<FunctionDecl>(E->getCalleeDecl()), Args);
844
845 return true;
846 }
847
848 void highlightMutableReferenceArgument(QualType T, const Expr *Arg) {
849 if (!Arg)
850 return;
851
852 // Is this parameter passed by non-const pointer or reference?
853 // FIXME The condition T->idDependentType() could be relaxed a bit,
854 // e.g. std::vector<T>& is dependent but we would want to highlight it
855 bool IsRef = T->isLValueReferenceType();
856 bool IsPtr = T->isPointerType();
857 if ((!IsRef && !IsPtr) || T->getPointeeType().isConstQualified() ||
858 T->isDependentType()) {
859 return;
860 }
861
862 std::optional<SourceLocation> Location;
863
864 // FIXME Add "unwrapping" for ArraySubscriptExpr,
865 // e.g. highlight `a` in `a[i]`
866 // FIXME Handle dependent expression types
867 if (auto *IC = dyn_cast<ImplicitCastExpr>(Arg))
868 Arg = IC->getSubExprAsWritten();
869 if (auto *UO = dyn_cast<UnaryOperator>(Arg)) {
870 if (UO->getOpcode() == UO_AddrOf)
871 Arg = UO->getSubExpr();
872 }
873 if (auto *DR = dyn_cast<DeclRefExpr>(Arg))
874 Location = DR->getLocation();
875 else if (auto *M = dyn_cast<MemberExpr>(Arg))
876 Location = M->getMemberLoc();
877
878 if (Location)
879 H.addExtraModifier(*Location,
880 IsRef ? HighlightingModifier::UsedAsMutableReference
881 : HighlightingModifier::UsedAsMutablePointer);
882 }
883
884 void
885 highlightMutableReferenceArguments(const FunctionDecl *FD,
886 llvm::ArrayRef<const Expr *const> Args) {
887 if (!FD)
888 return;
889
890 if (auto *ProtoType = FD->getType()->getAs<FunctionProtoType>()) {
891 // Iterate over the types of the function parameters.
892 // If any of them are non-const reference paramteres, add it as a
893 // highlighting modifier to the corresponding expression
894 for (size_t I = 0;
895 I < std::min(size_t(ProtoType->getNumParams()), Args.size()); ++I) {
896 highlightMutableReferenceArgument(ProtoType->getParamType(I), Args[I]);
897 }
898 }
899 }
900
901 bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
902 if (auto K = kindForType(L.getTypePtr(), H.getResolver())) {
903 auto &Tok = H.addToken(L.getBeginLoc(), *K)
904 .addModifier(HighlightingModifier::Deduced);
905 if (auto Mod = scopeModifier(L.getTypePtr()))
906 Tok.addModifier(*Mod);
907 if (isDefaultLibrary(L.getTypePtr()))
908 Tok.addModifier(HighlightingModifier::DefaultLibrary);
909 }
910 return true;
911 }
912
913 bool VisitCXXDestructorDecl(CXXDestructorDecl *D) {
914 if (auto *TI = D->getNameInfo().getNamedTypeInfo()) {
915 SourceLocation Loc = TI->getTypeLoc().getBeginLoc();
916 H.addExtraModifier(Loc, HighlightingModifier::ConstructorOrDestructor);
917 H.addExtraModifier(Loc, HighlightingModifier::Declaration);
918 if (D->isThisDeclarationADefinition())
919 H.addExtraModifier(Loc, HighlightingModifier::Definition);
920 }
921 return true;
922 }
923
924 bool VisitCXXMemberCallExpr(CXXMemberCallExpr *CE) {
925 // getMethodDecl can return nullptr with member pointers, e.g.
926 // `(foo.*pointer_to_member_fun)(arg);`
927 if (auto *D = CE->getMethodDecl()) {
928 if (isa<CXXDestructorDecl>(D)) {
929 if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee())) {
930 if (auto *TI = ME->getMemberNameInfo().getNamedTypeInfo()) {
931 H.addExtraModifier(TI->getTypeLoc().getBeginLoc(),
932 HighlightingModifier::ConstructorOrDestructor);
933 }
934 }
935 } else if (D->isOverloadedOperator()) {
936 if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee()))
937 H.addToken(
938 ME->getMemberNameInfo().getCXXOperatorNameRange().getBegin(),
939 HighlightingKind::Operator)
940 .addModifier(HighlightingModifier::UserDefined);
941 }
942 }
943 return true;
944 }
945
946 bool VisitDeclaratorDecl(DeclaratorDecl *D) {
947 for (unsigned i = 0; i < D->getNumTemplateParameterLists(); ++i) {
948 if (auto *TPL = D->getTemplateParameterList(i))
949 H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
950 }
951 auto *AT = D->getType()->getContainedAutoType();
952 if (!AT)
953 return true;
954 auto K =
955 kindForType(AT->getDeducedType().getTypePtrOrNull(), H.getResolver());
956 if (!K)
957 return true;
958 auto *TSI = D->getTypeSourceInfo();
959 if (!TSI)
960 return true;
961 SourceLocation StartLoc =
962 TSI->getTypeLoc().getContainedAutoTypeLoc().getNameLoc();
963 // The AutoType may not have a corresponding token, e.g. in the case of
964 // init-captures. In this case, StartLoc overlaps with the location
965 // of the decl itself, and producing a token for the type here would result
966 // in both it and the token for the decl being dropped due to conflict.
967 if (StartLoc == D->getLocation())
968 return true;
969
970 auto &Tok =
971 H.addToken(StartLoc, *K).addModifier(HighlightingModifier::Deduced);
972 const Type *Deduced = AT->getDeducedType().getTypePtrOrNull();
973 if (auto Mod = scopeModifier(Deduced))
974 Tok.addModifier(*Mod);
975 if (isDefaultLibrary(Deduced))
976 Tok.addModifier(HighlightingModifier::DefaultLibrary);
977 return true;
978 }
979
980 // We handle objective-C selectors specially, because one reference can
981 // cover several non-contiguous tokens.
982 void highlightObjCSelector(const ArrayRef<SourceLocation> &Locs, bool Decl,
983 bool Def, bool Class, bool DefaultLibrary) {
984 HighlightingKind Kind =
985 Class ? HighlightingKind::StaticMethod : HighlightingKind::Method;
986 for (SourceLocation Part : Locs) {
987 auto &Tok =
988 H.addToken(Part, Kind).addModifier(HighlightingModifier::ClassScope);
989 if (Decl)
990 Tok.addModifier(HighlightingModifier::Declaration);
991 if (Def)
992 Tok.addModifier(HighlightingModifier::Definition);
993 if (Class)
994 Tok.addModifier(HighlightingModifier::Static);
995 if (DefaultLibrary)
996 Tok.addModifier(HighlightingModifier::DefaultLibrary);
997 }
998 }
999
1000 bool VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
1001 llvm::SmallVector<SourceLocation> Locs;
1002 OMD->getSelectorLocs(Locs);
1003 highlightObjCSelector(Locs, /*Decl=*/true,
1004 OMD->isThisDeclarationADefinition(),
1005 OMD->isClassMethod(), isDefaultLibrary(OMD));
1006 return true;
1007 }
1008
1009 bool VisitObjCMessageExpr(ObjCMessageExpr *OME) {
1010 llvm::SmallVector<SourceLocation> Locs;
1011 OME->getSelectorLocs(Locs);
1012 bool DefaultLibrary = false;
1013 if (ObjCMethodDecl *OMD = OME->getMethodDecl())
1014 DefaultLibrary = isDefaultLibrary(OMD);
1015 highlightObjCSelector(Locs, /*Decl=*/false, /*Def=*/false,
1016 OME->isClassMessage(), DefaultLibrary);
1017 return true;
1018 }
1019
1020 // Objective-C allows you to use property syntax `self.prop` as sugar for
1021 // `[self prop]` and `[self setProp:]` when there's no explicit `@property`
1022 // for `prop` as well as for class properties. We treat this like a property
1023 // even though semantically it's equivalent to a method expression.
1024 void highlightObjCImplicitPropertyRef(const ObjCMethodDecl *OMD,
1025 SourceLocation Loc) {
1026 auto &Tok = H.addToken(Loc, HighlightingKind::Field)
1027 .addModifier(HighlightingModifier::ClassScope);
1028 if (OMD->isClassMethod())
1029 Tok.addModifier(HighlightingModifier::Static);
1030 if (isDefaultLibrary(OMD))
1031 Tok.addModifier(HighlightingModifier::DefaultLibrary);
1032 }
1033
1034 bool VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *OPRE) {
1035 // We need to handle implicit properties here since they will appear to
1036 // reference `ObjCMethodDecl` via an implicit `ObjCMessageExpr`, so normal
1037 // highlighting will not work.
1038 if (!OPRE->isImplicitProperty())
1039 return true;
1040 // A single property expr can reference both a getter and setter, but we can
1041 // only provide a single semantic token, so prefer the getter. In most cases
1042 // the end result should be the same, although it's technically possible
1043 // that the user defines a setter for a system SDK.
1044 if (OPRE->isMessagingGetter()) {
1045 highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertyGetter(),
1046 OPRE->getLocation());
1047 return true;
1048 }
1049 if (OPRE->isMessagingSetter()) {
1050 highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertySetter(),
1051 OPRE->getLocation());
1052 }
1053 return true;
1054 }
1055
1056 bool VisitOverloadExpr(OverloadExpr *E) {
1057 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
1058 if (!E->decls().empty())
1059 return true; // handled by findExplicitReferences.
1060 auto &Tok = H.addToken(E->getNameLoc(), HighlightingKind::Unknown)
1061 .addModifier(HighlightingModifier::DependentName);
1062 if (llvm::isa<UnresolvedMemberExpr>(E))
1063 Tok.addModifier(HighlightingModifier::ClassScope);
1064 // other case is UnresolvedLookupExpr, scope is unknown.
1065 return true;
1066 }
1067
1068 bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
1069 H.addToken(E->getMemberNameInfo().getLoc(), HighlightingKind::Unknown)
1070 .addModifier(HighlightingModifier::DependentName)
1071 .addModifier(HighlightingModifier::ClassScope);
1072 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
1073 return true;
1074 }
1075
1076 bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
1077 H.addToken(E->getNameInfo().getLoc(), HighlightingKind::Unknown)
1078 .addModifier(HighlightingModifier::DependentName)
1079 .addModifier(HighlightingModifier::ClassScope);
1080 H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
1081 return true;
1082 }
1083
1084 bool VisitAttr(Attr *A) {
1085 switch (A->getKind()) {
1086 case attr::Override:
1087 case attr::Final:
1088 H.addToken(A->getLocation(), HighlightingKind::Modifier);
1089 break;
1090 default:
1091 break;
1092 }
1093 return true;
1094 }
1095
1096 bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
1097 H.addToken(L.getNameLoc(), HighlightingKind::Type)
1098 .addModifier(HighlightingModifier::DependentName)
1099 .addModifier(HighlightingModifier::ClassScope);
1100 return true;
1101 }
1102
1103 bool VisitDependentTemplateSpecializationTypeLoc(
1104 DependentTemplateSpecializationTypeLoc L) {
1105 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Type)
1106 .addModifier(HighlightingModifier::DependentName)
1107 .addModifier(HighlightingModifier::ClassScope);
1108 H.addAngleBracketTokens(L.getLAngleLoc(), L.getRAngleLoc());
1109 return true;
1110 }
1111
1112 bool TraverseTemplateArgumentLoc(TemplateArgumentLoc L) {
1113 // Handle template template arguments only (other arguments are handled by
1114 // their Expr, TypeLoc etc values).
1115 if (L.getArgument().getKind() != TemplateArgument::Template &&
1116 L.getArgument().getKind() != TemplateArgument::TemplateExpansion)
1117 return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
1118
1119 TemplateName N = L.getArgument().getAsTemplateOrTemplatePattern();
1120 switch (N.getKind()) {
1121 case TemplateName::OverloadedTemplate:
1122 // Template template params must always be class templates.
1123 // Don't bother to try to work out the scope here.
1124 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class);
1125 break;
1126 case TemplateName::DependentTemplate:
1127 case TemplateName::AssumedTemplate:
1128 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class)
1129 .addModifier(HighlightingModifier::DependentName);
1130 break;
1131 case TemplateName::Template:
1132 case TemplateName::QualifiedTemplate:
1133 case TemplateName::SubstTemplateTemplateParm:
1134 case TemplateName::SubstTemplateTemplateParmPack:
1135 case TemplateName::UsingTemplate:
1136 // Names that could be resolved to a TemplateDecl are handled elsewhere.
1137 break;
1138 }
1139 return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
1140 }
1141
1142 // findExplicitReferences will walk nested-name-specifiers and
1143 // find anything that can be resolved to a Decl. However, non-leaf
1144 // components of nested-name-specifiers which are dependent names
1145 // (kind "Identifier") cannot be resolved to a decl, so we visit
1146 // them here.
1147 bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
1148 if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
1149 if (NNS->getKind() == NestedNameSpecifier::Identifier)
1150 H.addToken(Q.getLocalBeginLoc(), HighlightingKind::Type)
1151 .addModifier(HighlightingModifier::DependentName)
1152 .addModifier(HighlightingModifier::ClassScope);
1153 }
1154 return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
1155 }
1156
1157private:
1158 HighlightingsBuilder &H;
1159};
1160} // namespace
1161
1162std::vector<HighlightingToken>
1163getSemanticHighlightings(ParsedAST &AST, bool IncludeInactiveRegionTokens) {
1164 auto &C = AST.getASTContext();
1165 HighlightingFilter Filter = HighlightingFilter::fromCurrentConfig();
1166 if (!IncludeInactiveRegionTokens)
1167 Filter.disableKind(HighlightingKind::InactiveCode);
1168 // Add highlightings for AST nodes.
1169 HighlightingsBuilder Builder(AST, Filter);
1170 // Highlight 'decltype' and 'auto' as their underlying types.
1171 CollectExtraHighlightings(Builder).TraverseAST(C);
1172 // Highlight all decls and references coming from the AST.
1173 findExplicitReferences(
1174 C,
1175 [&](ReferenceLoc R) {
1176 for (const NamedDecl *Decl : R.Targets) {
1177 if (!canHighlightName(Decl->getDeclName()))
1178 continue;
1179 auto Kind = kindForDecl(Decl, AST.getHeuristicResolver());
1180 if (!Kind)
1181 continue;
1182 auto &Tok = Builder.addToken(R.NameLoc, *Kind);
1183
1184 // The attribute tests don't want to look at the template.
1185 if (auto *TD = dyn_cast<TemplateDecl>(Decl)) {
1186 if (auto *Templated = TD->getTemplatedDecl())
1187 Decl = Templated;
1188 }
1189 if (auto Mod = scopeModifier(Decl))
1190 Tok.addModifier(*Mod);
1191 if (isConst(Decl))
1192 Tok.addModifier(HighlightingModifier::Readonly);
1193 if (isStatic(Decl))
1194 Tok.addModifier(HighlightingModifier::Static);
1195 if (isAbstract(Decl))
1196 Tok.addModifier(HighlightingModifier::Abstract);
1197 if (isVirtual(Decl))
1198 Tok.addModifier(HighlightingModifier::Virtual);
1199 if (isDependent(Decl))
1200 Tok.addModifier(HighlightingModifier::DependentName);
1201 if (isDefaultLibrary(Decl))
1202 Tok.addModifier(HighlightingModifier::DefaultLibrary);
1203 if (Decl->isDeprecated())
1204 Tok.addModifier(HighlightingModifier::Deprecated);
1205 if (isa<CXXConstructorDecl>(Decl))
1206 Tok.addModifier(HighlightingModifier::ConstructorOrDestructor);
1207 if (R.IsDecl) {
1208 // Do not treat an UnresolvedUsingValueDecl as a declaration.
1209 // It's more common to think of it as a reference to the
1210 // underlying declaration.
1211 if (!isa<UnresolvedUsingValueDecl>(Decl))
1212 Tok.addModifier(HighlightingModifier::Declaration);
1213 if (isUniqueDefinition(Decl))
1214 Tok.addModifier(HighlightingModifier::Definition);
1215 }
1216 }
1217 },
1218 AST.getHeuristicResolver());
1219 // Add highlightings for macro references.
1220 auto AddMacro = [&](const MacroOccurrence &M) {
1221 auto &T = Builder.addToken(M.toRange(C.getSourceManager()),
1222 HighlightingKind::Macro);
1223 T.addModifier(HighlightingModifier::GlobalScope);
1224 if (M.IsDefinition)
1225 T.addModifier(HighlightingModifier::Declaration);
1226 };
1227 for (const auto &SIDToRefs : AST.getMacros().MacroRefs)
1228 for (const auto &M : SIDToRefs.second)
1229 AddMacro(M);
1230 for (const auto &M : AST.getMacros().UnknownMacros)
1231 AddMacro(M);
1232
1233 return std::move(Builder).collect(AST);
1234}
1235
1236llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
1237 switch (K) {
1238 case HighlightingKind::Variable:
1239 return OS << "Variable";
1240 case HighlightingKind::LocalVariable:
1241 return OS << "LocalVariable";
1242 case HighlightingKind::Parameter:
1243 return OS << "Parameter";
1244 case HighlightingKind::Function:
1245 return OS << "Function";
1246 case HighlightingKind::Method:
1247 return OS << "Method";
1248 case HighlightingKind::StaticMethod:
1249 return OS << "StaticMethod";
1250 case HighlightingKind::Field:
1251 return OS << "Field";
1252 case HighlightingKind::StaticField:
1253 return OS << "StaticField";
1254 case HighlightingKind::Class:
1255 return OS << "Class";
1256 case HighlightingKind::Interface:
1257 return OS << "Interface";
1258 case HighlightingKind::Enum:
1259 return OS << "Enum";
1260 case HighlightingKind::EnumConstant:
1261 return OS << "EnumConstant";
1262 case HighlightingKind::Typedef:
1263 return OS << "Typedef";
1264 case HighlightingKind::Type:
1265 return OS << "Type";
1266 case HighlightingKind::Unknown:
1267 return OS << "Unknown";
1268 case HighlightingKind::Namespace:
1269 return OS << "Namespace";
1270 case HighlightingKind::TemplateParameter:
1271 return OS << "TemplateParameter";
1272 case HighlightingKind::Concept:
1273 return OS << "Concept";
1274 case HighlightingKind::Primitive:
1275 return OS << "Primitive";
1276 case HighlightingKind::Macro:
1277 return OS << "Macro";
1278 case HighlightingKind::Modifier:
1279 return OS << "Modifier";
1280 case HighlightingKind::Operator:
1281 return OS << "Operator";
1282 case HighlightingKind::Bracket:
1283 return OS << "Bracket";
1284 case HighlightingKind::Label:
1285 return OS << "Label";
1286 case HighlightingKind::InactiveCode:
1287 return OS << "InactiveCode";
1288 }
1289 llvm_unreachable("invalid HighlightingKind");
1290}
1291std::optional<HighlightingKind>
1292highlightingKindFromString(llvm::StringRef Name) {
1293 static llvm::StringMap<HighlightingKind> Lookup = {
1294 {"Variable", HighlightingKind::Variable},
1295 {"LocalVariable", HighlightingKind::LocalVariable},
1296 {"Parameter", HighlightingKind::Parameter},
1297 {"Function", HighlightingKind::Function},
1298 {"Method", HighlightingKind::Method},
1299 {"StaticMethod", HighlightingKind::StaticMethod},
1300 {"Field", HighlightingKind::Field},
1301 {"StaticField", HighlightingKind::StaticField},
1302 {"Class", HighlightingKind::Class},
1303 {"Interface", HighlightingKind::Interface},
1304 {"Enum", HighlightingKind::Enum},
1305 {"EnumConstant", HighlightingKind::EnumConstant},
1306 {"Typedef", HighlightingKind::Typedef},
1307 {"Type", HighlightingKind::Type},
1308 {"Unknown", HighlightingKind::Unknown},
1309 {"Namespace", HighlightingKind::Namespace},
1310 {"TemplateParameter", HighlightingKind::TemplateParameter},
1311 {"Concept", HighlightingKind::Concept},
1312 {"Primitive", HighlightingKind::Primitive},
1313 {"Macro", HighlightingKind::Macro},
1314 {"Modifier", HighlightingKind::Modifier},
1315 {"Operator", HighlightingKind::Operator},
1316 {"Bracket", HighlightingKind::Bracket},
1317 {"InactiveCode", HighlightingKind::InactiveCode},
1318 };
1319
1320 auto It = Lookup.find(Name);
1321 return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
1322}
1323llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
1324 switch (K) {
1325 case HighlightingModifier::Declaration:
1326 return OS << "decl"; // abbreviation for common case
1327 case HighlightingModifier::Definition:
1328 return OS << "def"; // abbrevation for common case
1329 case HighlightingModifier::ConstructorOrDestructor:
1330 return OS << "constrDestr";
1331 default:
1332 return OS << toSemanticTokenModifier(K);
1333 }
1334}
1335std::optional<HighlightingModifier>
1336highlightingModifierFromString(llvm::StringRef Name) {
1337 static llvm::StringMap<HighlightingModifier> Lookup = {
1338 {"Declaration", HighlightingModifier::Declaration},
1339 {"Definition", HighlightingModifier::Definition},
1340 {"Deprecated", HighlightingModifier::Deprecated},
1341 {"Deduced", HighlightingModifier::Deduced},
1342 {"Readonly", HighlightingModifier::Readonly},
1343 {"Static", HighlightingModifier::Static},
1344 {"Abstract", HighlightingModifier::Abstract},
1345 {"Virtual", HighlightingModifier::Virtual},
1346 {"DependentName", HighlightingModifier::DependentName},
1347 {"DefaultLibrary", HighlightingModifier::DefaultLibrary},
1348 {"UsedAsMutableReference", HighlightingModifier::UsedAsMutableReference},
1349 {"UsedAsMutablePointer", HighlightingModifier::UsedAsMutablePointer},
1350 {"ConstructorOrDestructor",
1351 HighlightingModifier::ConstructorOrDestructor},
1352 {"UserDefined", HighlightingModifier::UserDefined},
1353 {"FunctionScope", HighlightingModifier::FunctionScope},
1354 {"ClassScope", HighlightingModifier::ClassScope},
1355 {"FileScope", HighlightingModifier::FileScope},
1356 {"GlobalScope", HighlightingModifier::GlobalScope},
1357 };
1358
1359 auto It = Lookup.find(Name);
1360 return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
1361}
1362
1363bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
1364 return std::tie(L.R, L.Kind, L.Modifiers) ==
1365 std::tie(R.R, R.Kind, R.Modifiers);
1366}
1367bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
1368 return std::tie(L.R, L.Kind, L.Modifiers) <
1369 std::tie(R.R, R.Kind, R.Modifiers);
1370}
1371
1372std::vector<SemanticToken>
1373toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens,
1374 llvm::StringRef Code) {
1375 assert(llvm::is_sorted(Tokens));
1376 std::vector<SemanticToken> Result;
1377 // In case we split a HighlightingToken into multiple tokens (e.g. because it
1378 // was spanning multiple lines), this tracks the last one. This prevents
1379 // having a copy all the time.
1380 HighlightingToken Scratch;
1381 const HighlightingToken *Last = nullptr;
1382 for (const HighlightingToken &Tok : Tokens) {
1383 Result.emplace_back();
1384 SemanticToken *Out = &Result.back();
1385 // deltaStart/deltaLine are relative if possible.
1386 if (Last) {
1387 assert(Tok.R.start.line >= Last->R.end.line);
1388 Out->deltaLine = Tok.R.start.line - Last->R.end.line;
1389 if (Out->deltaLine == 0) {
1390 assert(Tok.R.start.character >= Last->R.start.character);
1391 Out->deltaStart = Tok.R.start.character - Last->R.start.character;
1392 } else {
1393 Out->deltaStart = Tok.R.start.character;
1394 }
1395 } else {
1396 Out->deltaLine = Tok.R.start.line;
1397 Out->deltaStart = Tok.R.start.character;
1398 }
1399 Out->tokenType = static_cast<unsigned>(Tok.Kind);
1400 Out->tokenModifiers = Tok.Modifiers;
1401 Last = &Tok;
1402
1403 if (Tok.R.end.line == Tok.R.start.line) {
1404 Out->length = Tok.R.end.character - Tok.R.start.character;
1405 } else {
1406 // If the token spans a line break, split it into multiple pieces for each
1407 // line.
1408 // This is slow, but multiline tokens are rare.
1409 // FIXME: There's a client capability for supporting multiline tokens,
1410 // respect that.
1411 auto TokStartOffset = llvm::cantFail(positionToOffset(Code, Tok.R.start));
1412 // Note that the loop doesn't cover the last line, which has a special
1413 // length.
1414 for (int I = Tok.R.start.line; I < Tok.R.end.line; ++I) {
1415 auto LineEnd = Code.find('\n', TokStartOffset);
1416 assert(LineEnd != Code.npos);
1417 Out->length = LineEnd - TokStartOffset;
1418 // Token continues on next line, right after the line break.
1419 TokStartOffset = LineEnd + 1;
1420 Result.emplace_back();
1421 Out = &Result.back();
1422 *Out = Result[Result.size() - 2];
1423 // New token starts at the first column of the next line.
1424 Out->deltaLine = 1;
1425 Out->deltaStart = 0;
1426 }
1427 // This is the token on last line.
1428 Out->length = Tok.R.end.character;
1429 // Update the start location for last token, as that's used in the
1430 // relative delta calculation for following tokens.
1431 Scratch = *Last;
1432 Scratch.R.start.line = Tok.R.end.line;
1433 Scratch.R.start.character = 0;
1434 Last = &Scratch;
1435 }
1436 }
1437 return Result;
1438}
1439llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {
1440 switch (Kind) {
1441 case HighlightingKind::Variable:
1442 case HighlightingKind::LocalVariable:
1443 case HighlightingKind::StaticField:
1444 return "variable";
1445 case HighlightingKind::Parameter:
1446 return "parameter";
1447 case HighlightingKind::Function:
1448 return "function";
1449 case HighlightingKind::Method:
1450 return "method";
1451 case HighlightingKind::StaticMethod:
1452 // FIXME: better method with static modifier?
1453 return "function";
1454 case HighlightingKind::Field:
1455 return "property";
1456 case HighlightingKind::Class:
1457 return "class";
1458 case HighlightingKind::Interface:
1459 return "interface";
1460 case HighlightingKind::Enum:
1461 return "enum";
1462 case HighlightingKind::EnumConstant:
1463 return "enumMember";
1464 case HighlightingKind::Typedef:
1465 case HighlightingKind::Type:
1466 return "type";
1467 case HighlightingKind::Unknown:
1468 return "unknown"; // nonstandard
1469 case HighlightingKind::Namespace:
1470 return "namespace";
1471 case HighlightingKind::TemplateParameter:
1472 return "typeParameter";
1473 case HighlightingKind::Concept:
1474 return "concept"; // nonstandard
1475 case HighlightingKind::Primitive:
1476 return "type";
1477 case HighlightingKind::Macro:
1478 return "macro";
1479 case HighlightingKind::Modifier:
1480 return "modifier";
1481 case HighlightingKind::Operator:
1482 return "operator";
1483 case HighlightingKind::Bracket:
1484 return "bracket";
1485 case HighlightingKind::Label:
1486 return "label";
1487 case HighlightingKind::InactiveCode:
1488 return "comment";
1489 }
1490 llvm_unreachable("unhandled HighlightingKind");
1491}
1492
1493llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
1494 switch (Modifier) {
1495 case HighlightingModifier::Declaration:
1496 return "declaration";
1497 case HighlightingModifier::Definition:
1498 return "definition";
1499 case HighlightingModifier::Deprecated:
1500 return "deprecated";
1501 case HighlightingModifier::Readonly:
1502 return "readonly";
1503 case HighlightingModifier::Static:
1504 return "static";
1505 case HighlightingModifier::Deduced:
1506 return "deduced"; // nonstandard
1507 case HighlightingModifier::Abstract:
1508 return "abstract";
1509 case HighlightingModifier::Virtual:
1510 return "virtual";
1511 case HighlightingModifier::DependentName:
1512 return "dependentName"; // nonstandard
1513 case HighlightingModifier::DefaultLibrary:
1514 return "defaultLibrary";
1515 case HighlightingModifier::UsedAsMutableReference:
1516 return "usedAsMutableReference"; // nonstandard
1517 case HighlightingModifier::UsedAsMutablePointer:
1518 return "usedAsMutablePointer"; // nonstandard
1519 case HighlightingModifier::ConstructorOrDestructor:
1520 return "constructorOrDestructor"; // nonstandard
1521 case HighlightingModifier::UserDefined:
1522 return "userDefined"; // nonstandard
1523 case HighlightingModifier::FunctionScope:
1524 return "functionScope"; // nonstandard
1525 case HighlightingModifier::ClassScope:
1526 return "classScope"; // nonstandard
1527 case HighlightingModifier::FileScope:
1528 return "fileScope"; // nonstandard
1529 case HighlightingModifier::GlobalScope:
1530 return "globalScope"; // nonstandard
1531 }
1532 llvm_unreachable("unhandled HighlightingModifier");
1533}
1534
1535std::vector<SemanticTokensEdit>
1536diffTokens(llvm::ArrayRef<SemanticToken> Old,
1537 llvm::ArrayRef<SemanticToken> New) {
1538 // For now, just replace everything from the first-last modification.
1539 // FIXME: use a real diff instead, this is bad with include-insertion.
1540
1541 unsigned Offset = 0;
1542 while (!Old.empty() && !New.empty() && Old.front() == New.front()) {
1543 ++Offset;
1544 Old = Old.drop_front();
1545 New = New.drop_front();
1546 }
1547 while (!Old.empty() && !New.empty() && Old.back() == New.back()) {
1548 Old = Old.drop_back();
1549 New = New.drop_back();
1550 }
1551
1552 if (Old.empty() && New.empty())
1553 return {};
1554 SemanticTokensEdit Edit;
1555 Edit.startToken = Offset;
1556 Edit.deleteTokens = Old.size();
1557 Edit.tokens = New;
1558 return {std::move(Edit)};
1559}
1560
1561std::vector<Range> getInactiveRegions(ParsedAST &AST) {
1562 std::vector<Range> SkippedRanges(std::move(AST.getMacros().SkippedRanges));
1563 const auto &SM = AST.getSourceManager();
1564 StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
1565 std::vector<Range> InactiveRegions;
1566 for (const Range &Skipped : SkippedRanges) {
1567 Range Inactive = Skipped;
1568 // Sometimes, SkippedRanges contains a range ending at position 0
1569 // of a line. Clients that apply whole-line styles will treat that
1570 // line as inactive which is not desirable, so adjust the ending
1571 // position to be the end of the previous line.
1572 if (Inactive.end.character == 0 && Inactive.end.line > 0) {
1573 --Inactive.end.line;
1574 }
1575 // Exclude the directive lines themselves from the range.
1576 if (Inactive.end.line >= Inactive.start.line + 2) {
1577 ++Inactive.start.line;
1578 --Inactive.end.line;
1579 } else {
1580 // range would be empty, e.g. #endif on next line after #ifdef
1581 continue;
1582 }
1583 // Since we've adjusted the ending line, we need to recompute the
1584 // column to reflect the end of that line.
1585 if (auto EndOfLine = endOfLine(MainCode, Inactive.end.line)) {
1586 Inactive.end = *EndOfLine;
1587 } else {
1588 elog("Failed to determine end of line: {0}", EndOfLine.takeError());
1589 continue;
1590 }
1591 InactiveRegions.push_back(Inactive);
1592 }
1593 return InactiveRegions;
1594}
1595
1596} // namespace clangd
1597} // namespace clang
1598