Polly 24.0.0git
ScopDetection.cpp
Go to the documentation of this file.
1//===- ScopDetection.cpp - Detect Scops -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Detect the maximal Scops of a function.
10//
11// A static control part (Scop) is a subgraph of the control flow graph (CFG)
12// that only has statically known control flow and can therefore be described
13// within the polyhedral model.
14//
15// Every Scop fulfills these restrictions:
16//
17// * It is a single entry single exit region
18//
19// * Only affine linear bounds in the loops
20//
21// Every natural loop in a Scop must have a number of loop iterations that can
22// be described as an affine linear function in surrounding loop iterators or
23// parameters. (A parameter is a scalar that does not change its value during
24// execution of the Scop).
25//
26// * Only comparisons of affine linear expressions in conditions
27//
28// * All loops and conditions perfectly nested
29//
30// The control flow needs to be structured such that it could be written using
31// just 'for' and 'if' statements, without the need for any 'goto', 'break' or
32// 'continue'.
33//
34// * Side effect free functions call
35//
36// Function calls and intrinsics that do not have side effects (readnone)
37// or memory intrinsics (memset, memcpy, memmove) are allowed.
38//
39// The Scop detection finds the largest Scops by checking if the largest
40// region is a Scop. If this is not the case, its canonical subregions are
41// checked until a region is a Scop. It is now tried to extend this Scop by
42// creating a larger non canonical region.
43//
44//===----------------------------------------------------------------------===//
45
46#include "polly/ScopDetection.h"
47#include "polly/Options.h"
52#include "llvm/ADT/SmallPtrSet.h"
53#include "llvm/ADT/Statistic.h"
54#include "llvm/Analysis/AliasAnalysis.h"
55#include "llvm/Analysis/Delinearization.h"
56#include "llvm/Analysis/Loads.h"
57#include "llvm/Analysis/LoopInfo.h"
58#include "llvm/Analysis/OptimizationRemarkEmitter.h"
59#include "llvm/Analysis/RegionInfo.h"
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/Analysis/ScalarEvolutionExpressions.h"
62#include "llvm/IR/BasicBlock.h"
63#include "llvm/IR/DebugLoc.h"
64#include "llvm/IR/DerivedTypes.h"
65#include "llvm/IR/DiagnosticInfo.h"
66#include "llvm/IR/DiagnosticPrinter.h"
67#include "llvm/IR/Dominators.h"
68#include "llvm/IR/Function.h"
69#include "llvm/IR/InstrTypes.h"
70#include "llvm/IR/Instruction.h"
71#include "llvm/IR/Instructions.h"
72#include "llvm/IR/IntrinsicInst.h"
73#include "llvm/IR/Metadata.h"
74#include "llvm/IR/Module.h"
75#include "llvm/IR/Value.h"
76#include "llvm/Support/Debug.h"
77#include "llvm/Support/Regex.h"
78#include "llvm/Support/raw_ostream.h"
79#include <algorithm>
80#include <cassert>
81#include <memory>
82#include <stack>
83#include <string>
84#include <utility>
85#include <vector>
86
87using namespace llvm;
88using namespace polly;
89
91#define DEBUG_TYPE "polly-detect"
92
93// This option is set to a very high value, as analyzing such loops increases
94// compile time on several cases. For experiments that enable this option,
95// a value of around 40 has been working to avoid run-time regressions with
96// Polly while still exposing interesting optimization opportunities.
98 "polly-detect-profitability-min-per-loop-insts",
99 cl::desc("The minimal number of per-loop instructions before a single loop "
100 "region is considered profitable"),
101 cl::Hidden, cl::ValueRequired, cl::init(100000000), cl::cat(PollyCategory));
102
104
105static cl::opt<bool, true> XPollyProcessUnprofitable(
106 "polly-process-unprofitable",
107 cl::desc(
108 "Process scops that are unlikely to benefit from Polly optimizations."),
109 cl::location(PollyProcessUnprofitable), cl::cat(PollyCategory));
110
111static cl::list<std::string> OnlyFunctions(
112 "polly-only-func",
113 cl::desc("Only run on functions that match a regex. "
114 "Multiple regexes can be comma separated. "
115 "Scop detection will run on all functions that match "
116 "ANY of the regexes provided."),
117 cl::CommaSeparated, cl::cat(PollyCategory));
118
119static cl::list<std::string> IgnoredFunctions(
120 "polly-ignore-func",
121 cl::desc("Ignore functions that match a regex. "
122 "Multiple regexes can be comma separated. "
123 "Scop detection will ignore all functions that match "
124 "ANY of the regexes provided."),
125 cl::CommaSeparated, cl::cat(PollyCategory));
126
128
129static cl::opt<bool, true>
130 XAllowFullFunction("polly-detect-full-functions",
131 cl::desc("Allow the detection of full functions"),
132 cl::location(polly::PollyAllowFullFunction),
133 cl::init(false), cl::cat(PollyCategory));
134
135static cl::opt<std::string> OnlyRegion(
136 "polly-only-region",
137 cl::desc("Only run on certain regions (The provided identifier must "
138 "appear in the name of the region's entry block"),
139 cl::value_desc("identifier"), cl::ValueRequired, cl::init(""),
140 cl::cat(PollyCategory));
141
142static cl::opt<bool>
143 IgnoreAliasing("polly-ignore-aliasing",
144 cl::desc("Ignore possible aliasing of the array bases"),
145 cl::Hidden, cl::cat(PollyCategory));
146
148
149static cl::opt<bool, true> XPollyAllowUnsignedOperations(
150 "polly-allow-unsigned-operations",
151 cl::desc("Allow unsigned operations such as comparisons or zero-extends."),
152 cl::location(PollyAllowUnsignedOperations), cl::Hidden, cl::init(true),
153 cl::cat(PollyCategory));
154
156
157static cl::opt<bool, true> XPollyUseRuntimeAliasChecks(
158 "polly-use-runtime-alias-checks",
159 cl::desc("Use runtime alias checks to resolve possible aliasing."),
160 cl::location(PollyUseRuntimeAliasChecks), cl::Hidden, cl::init(true),
161 cl::cat(PollyCategory));
162
163static cl::opt<bool>
164 ReportLevel("polly-report",
165 cl::desc("Print information about the activities of Polly"),
166 cl::cat(PollyCategory));
167
168static cl::opt<bool> AllowDifferentTypes(
169 "polly-allow-differing-element-types",
170 cl::desc("Allow different element types for array accesses"), cl::Hidden,
171 cl::init(true), cl::cat(PollyCategory));
172
173static cl::opt<bool>
174 AllowNonAffine("polly-allow-nonaffine",
175 cl::desc("Allow non affine access functions in arrays"),
176 cl::Hidden, cl::cat(PollyCategory));
177
178static cl::opt<bool>
179 AllowModrefCall("polly-allow-modref-calls",
180 cl::desc("Allow functions with known modref behavior"),
181 cl::Hidden, cl::cat(PollyCategory));
182
183static cl::opt<bool> AllowNonAffineSubRegions(
184 "polly-allow-nonaffine-branches",
185 cl::desc("Allow non affine conditions for branches"), cl::Hidden,
186 cl::init(true), cl::cat(PollyCategory));
187
188static cl::opt<bool>
189 AllowNonAffineSubLoops("polly-allow-nonaffine-loops",
190 cl::desc("Allow non affine conditions for loops"),
191 cl::Hidden, cl::cat(PollyCategory));
192
193static cl::opt<bool, true>
194 TrackFailures("polly-detect-track-failures",
195 cl::desc("Track failure strings in detecting scop regions"),
196 cl::location(PollyTrackFailures), cl::Hidden, cl::init(true),
197 cl::cat(PollyCategory));
198
199static cl::opt<bool> KeepGoing("polly-detect-keep-going",
200 cl::desc("Do not fail on the first error."),
201 cl::Hidden, cl::cat(PollyCategory));
202
203static cl::opt<bool, true>
204 PollyDelinearizeX("polly-delinearize",
205 cl::desc("Delinearize array access functions"),
206 cl::location(PollyDelinearize), cl::Hidden,
207 cl::init(true), cl::cat(PollyCategory));
208
209static cl::opt<bool>
210 VerifyScops("polly-detect-verify",
211 cl::desc("Verify the detected SCoPs after each transformation"),
212 cl::Hidden, cl::cat(PollyCategory));
213
215
216static cl::opt<bool, true>
217 XPollyInvariantLoadHoisting("polly-invariant-load-hoisting",
218 cl::desc("Hoist invariant loads."),
219 cl::location(PollyInvariantLoadHoisting),
220 cl::Hidden, cl::cat(PollyCategory));
221
222static cl::opt<bool> PollyAllowErrorBlocks(
223 "polly-allow-error-blocks",
224 cl::desc("Allow to speculate on the execution of 'error blocks'."),
225 cl::Hidden, cl::init(true), cl::cat(PollyCategory));
226
227/// The minimal trip count under which loops are considered unprofitable.
228static const unsigned MIN_LOOP_TRIP_COUNT = 8;
229
232StringRef polly::PollySkipFnAttr = "polly.skip.fn";
233
234//===----------------------------------------------------------------------===//
235// Statistics.
236
237STATISTIC(NumScopRegions, "Number of scops");
238STATISTIC(NumLoopsInScop, "Number of loops in scops");
239STATISTIC(NumScopsDepthZero, "Number of scops with maximal loop depth 0");
240STATISTIC(NumScopsDepthOne, "Number of scops with maximal loop depth 1");
241STATISTIC(NumScopsDepthTwo, "Number of scops with maximal loop depth 2");
242STATISTIC(NumScopsDepthThree, "Number of scops with maximal loop depth 3");
243STATISTIC(NumScopsDepthFour, "Number of scops with maximal loop depth 4");
244STATISTIC(NumScopsDepthFive, "Number of scops with maximal loop depth 5");
245STATISTIC(NumScopsDepthLarger,
246 "Number of scops with maximal loop depth 6 and larger");
247STATISTIC(NumProfScopRegions, "Number of scops (profitable scops only)");
248STATISTIC(NumLoopsInProfScop,
249 "Number of loops in scops (profitable scops only)");
250STATISTIC(NumLoopsOverall, "Number of total loops");
251STATISTIC(NumProfScopsDepthZero,
252 "Number of scops with maximal loop depth 0 (profitable scops only)");
253STATISTIC(NumProfScopsDepthOne,
254 "Number of scops with maximal loop depth 1 (profitable scops only)");
255STATISTIC(NumProfScopsDepthTwo,
256 "Number of scops with maximal loop depth 2 (profitable scops only)");
257STATISTIC(NumProfScopsDepthThree,
258 "Number of scops with maximal loop depth 3 (profitable scops only)");
259STATISTIC(NumProfScopsDepthFour,
260 "Number of scops with maximal loop depth 4 (profitable scops only)");
261STATISTIC(NumProfScopsDepthFive,
262 "Number of scops with maximal loop depth 5 (profitable scops only)");
263STATISTIC(NumProfScopsDepthLarger,
264 "Number of scops with maximal loop depth 6 and larger "
265 "(profitable scops only)");
266STATISTIC(MaxNumLoopsInScop, "Maximal number of loops in scops");
267STATISTIC(MaxNumLoopsInProfScop,
268 "Maximal number of loops in scops (profitable scops only)");
269
271 bool OnlyProfitable);
272
273namespace {
274
275class DiagnosticScopFound final : public DiagnosticInfo {
276private:
277 static int PluginDiagnosticKind;
278
279 Function &F;
280 std::string FileName;
281 unsigned EntryLine, ExitLine;
282
283public:
284 DiagnosticScopFound(Function &F, std::string FileName, unsigned EntryLine,
285 unsigned ExitLine)
286 : DiagnosticInfo(PluginDiagnosticKind, DS_Note), F(F), FileName(FileName),
287 EntryLine(EntryLine), ExitLine(ExitLine) {}
288
289 void print(DiagnosticPrinter &DP) const override;
290
291 static bool classof(const DiagnosticInfo *DI) {
292 return DI->getKind() == PluginDiagnosticKind;
293 }
294};
295} // namespace
296
297int DiagnosticScopFound::PluginDiagnosticKind =
298 getNextAvailablePluginDiagnosticKind();
299
300void DiagnosticScopFound::print(DiagnosticPrinter &DP) const {
301 DP << "Polly detected an optimizable loop region (scop) in function '" << F
302 << "'\n";
303
304 if (FileName.empty()) {
305 DP << "Scop location is unknown. Compile with debug info "
306 "(-g) to get more precise information. ";
307 return;
308 }
309
310 DP << FileName << ":" << EntryLine << ": Start of scop\n";
311 DP << FileName << ":" << ExitLine << ": End of scop";
312}
313
314/// Check if a string matches any regex in a list of regexes.
315/// @param Str the input string to match against.
316/// @param RegexList a list of strings that are regular expressions.
317static bool doesStringMatchAnyRegex(StringRef Str,
318 const cl::list<std::string> &RegexList) {
319 for (auto RegexStr : RegexList) {
320 Regex R(RegexStr);
321
322 std::string Err;
323 if (!R.isValid(Err))
324 report_fatal_error(Twine("invalid regex given as input to polly: ") + Err,
325 true);
326
327 if (R.match(Str))
328 return true;
329 }
330 return false;
331}
332
333//===----------------------------------------------------------------------===//
334// ScopDetection.
335
336ScopDetection::ScopDetection(const DominatorTree &DT, ScalarEvolution &SE,
337 LoopInfo &LI, RegionInfo &RI, AAResults &AA,
338 OptimizationRemarkEmitter &ORE)
339 : DT(DT), SE(SE), LI(LI), RI(RI), AA(AA), ORE(ORE) {}
340
341void ScopDetection::detect(Function &F) {
342 assert(ValidRegions.empty() && "Detection must run only once");
343
344 if (!PollyProcessUnprofitable && LI.empty())
345 return;
346
347 Region *TopRegion = RI.getTopLevelRegion();
348
349 if (!OnlyFunctions.empty() &&
351 return;
352
354 return;
355
356 if (!isValidFunction(F))
357 return;
358
359 findScops(*TopRegion);
360
361 NumScopRegions += ValidRegions.size();
362
363 // Prune non-profitable regions.
364 for (auto &DIt : DetectionContextMap) {
365 DetectionContext &DC = *DIt.getSecond();
366 if (DC.Log.hasErrors())
367 continue;
368 if (!ValidRegions.count(&DC.CurRegion))
369 continue;
370 LoopStats Stats = countBeneficialLoops(&DC.CurRegion, SE, LI, 0);
371 updateLoopCountStatistic(Stats, false /* OnlyProfitable */);
372 if (isProfitableRegion(DC)) {
373 updateLoopCountStatistic(Stats, true /* OnlyProfitable */);
374 continue;
375 }
376
377 ValidRegions.remove(&DC.CurRegion);
378 }
379
380 NumProfScopRegions += ValidRegions.size();
381 NumLoopsOverall += countBeneficialLoops(TopRegion, SE, LI, 0).NumLoops;
382
383 // Only makes sense when we tracked errors.
386
387 if (ReportLevel)
389
390 assert(ValidRegions.size() <= DetectionContextMap.size() &&
391 "Cached more results than valid regions");
392}
393
394template <class RR, typename... Args>
395inline bool ScopDetection::invalid(DetectionContext &Context, bool Assert,
396 Args &&...Arguments) const {
397 if (!Context.Verifying) {
398 RejectLog &Log = Context.Log;
399 std::shared_ptr<RR> RejectReason = std::make_shared<RR>(Arguments...);
400 Context.IsInvalid = true;
401
402 // Log even if PollyTrackFailures is false, the log entries are also used in
403 // canUseISLTripCount().
404 Log.report(RejectReason);
405
406 POLLY_DEBUG(dbgs() << RejectReason->getMessage());
407 POLLY_DEBUG(dbgs() << "\n");
408 } else {
409 assert(!Assert && "Verification of detected scop failed");
410 }
411
412 return false;
413}
414
415bool ScopDetection::isMaxRegionInScop(const Region &R, bool Verify) {
416 if (!ValidRegions.count(&R))
417 return false;
418
419 if (Verify) {
421 std::unique_ptr<DetectionContext> &Entry = DetectionContextMap[P];
422
423 // Free previous DetectionContext for the region and create and verify a new
424 // one. Be sure that the DetectionContext is not still used by a ScopInfop.
425 // Due to changes but CodeGeneration of another Scop, the Region object and
426 // the BBPair might not match anymore.
427 Entry = std::make_unique<DetectionContext>(const_cast<Region &>(R), AA,
428 /*Verifying=*/false);
429
430 return isValidRegion(*Entry);
431 }
432
433 return true;
434}
435
436std::string ScopDetection::regionIsInvalidBecause(const Region *R) const {
437 // Get the first error we found. Even in keep-going mode, this is the first
438 // reason that caused the candidate to be rejected.
439 auto *Log = lookupRejectionLog(R);
440
441 // This can happen when we marked a region invalid, but didn't track
442 // an error for it.
443 if (!Log || !Log->hasErrors())
444 return "";
445
446 RejectReasonPtr RR = *Log->begin();
447 return RR->getMessage();
448}
449
451 DetectionContext &Context) const {
452 // If we already know about Ar we can exit.
453 if (!Context.NonAffineSubRegionSet.insert(AR))
454 return true;
455
456 // All loops in the region have to be overapproximated too if there
457 // are accesses that depend on the iteration count.
458
459 for (BasicBlock *BB : AR->blocks()) {
460 Loop *L = LI.getLoopFor(BB);
461 if (AR->contains(L))
462 Context.BoxedLoopsSet.insert(L);
463 }
464
465 return (AllowNonAffineSubLoops || Context.BoxedLoopsSet.empty());
466}
467
469 InvariantLoadsSetTy &RequiredILS, DetectionContext &Context) const {
470 Region &CurRegion = Context.CurRegion;
471 const DataLayout &DL = CurRegion.getEntry()->getModule()->getDataLayout();
472
473 if (!PollyInvariantLoadHoisting && !RequiredILS.empty())
474 return false;
475
476 for (LoadInst *Load : RequiredILS) {
477 // If we already know a load has been accepted as required invariant, we
478 // already run the validation below once and consequently don't need to
479 // run it again. Hence, we return early. For certain test cases (e.g.,
480 // COSMO this avoids us spending 50% of scop-detection time in this
481 // very function (and its children).
482 if (Context.RequiredILS.count(Load))
483 continue;
484 if (!isHoistableLoad(Load, CurRegion, LI, SE, DT, Context.RequiredILS))
485 return false;
486
487 for (auto NonAffineRegion : Context.NonAffineSubRegionSet) {
488 if (isSafeToLoadUnconditionally(Load->getPointerOperand(),
489 Load->getType(), Load->getAlign(), DL))
490 continue;
491
492 if (NonAffineRegion->contains(Load) &&
493 Load->getParent() != NonAffineRegion->getEntry())
494 return false;
495 }
496 }
497
498 Context.RequiredILS.insert_range(RequiredILS);
499
500 return true;
501}
502
503bool ScopDetection::involvesMultiplePtrs(const SCEV *S0, const SCEV *S1,
504 Loop *Scope) const {
505 SetVector<Value *> Values;
506 findValues(S0, SE, Values);
507 if (S1)
508 findValues(S1, SE, Values);
509
510 SmallPtrSet<Value *, 8> PtrVals;
511 for (auto *V : Values) {
512 if (auto *P2I = dyn_cast<PtrToIntInst>(V))
513 V = P2I->getOperand(0);
514
515 if (!V->getType()->isPointerTy())
516 continue;
517
518 const SCEV *PtrSCEV = SE.getSCEVAtScope(V, Scope);
519 if (isa<SCEVConstant>(PtrSCEV))
520 continue;
521
522 auto *BasePtr = dyn_cast<SCEVUnknown>(SE.getPointerBase(PtrSCEV));
523 if (!BasePtr)
524 return true;
525
526 Value *BasePtrVal = BasePtr->getValue();
527 if (PtrVals.insert(BasePtrVal).second) {
528 for (auto *PtrVal : PtrVals)
529 if (PtrVal != BasePtrVal && !AA.isNoAlias(PtrVal, BasePtrVal))
530 return true;
531 }
532 }
533
534 return false;
535}
536
537bool ScopDetection::isAffine(const SCEV *S, Loop *Scope,
538 DetectionContext &Context) const {
539 InvariantLoadsSetTy AccessILS;
540 if (!isAffineExpr(&Context.CurRegion, Scope, S, SE, &AccessILS))
541 return false;
542
543 if (!onlyValidRequiredInvariantLoads(AccessILS, Context))
544 return false;
545
546 return true;
547}
548
549bool ScopDetection::isValidSwitch(BasicBlock &BB, SwitchInst *SI,
550 Value *Condition, bool IsLoopBranch,
551 DetectionContext &Context) const {
552 Loop *L = LI.getLoopFor(&BB);
553 const SCEV *ConditionSCEV = SE.getSCEVAtScope(Condition, L);
554
555 if (IsLoopBranch && L->isLoopLatch(&BB))
556 return false;
557
558 // Check for invalid usage of different pointers in one expression.
559 if (involvesMultiplePtrs(ConditionSCEV, nullptr, L))
560 return false;
561
562 if (isAffine(ConditionSCEV, L, Context))
563 return true;
564
566 addOverApproximatedRegion(RI.getRegionFor(&BB), Context))
567 return true;
568
569 return invalid<ReportNonAffBranch>(Context, /*Assert=*/true, &BB,
570 ConditionSCEV, ConditionSCEV, SI);
571}
572
573bool ScopDetection::isValidBranch(BasicBlock &BB, CondBrInst *BI,
574 Value *Condition, bool IsLoopBranch,
575 DetectionContext &Context) {
576 // Constant integer conditions are always affine.
577 if (isa<ConstantInt>(Condition))
578 return true;
579
580 if (BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Condition)) {
581 auto Opcode = BinOp->getOpcode();
582 if (Opcode == Instruction::And || Opcode == Instruction::Or) {
583 Value *Op0 = BinOp->getOperand(0);
584 Value *Op1 = BinOp->getOperand(1);
585 return isValidBranch(BB, BI, Op0, IsLoopBranch, Context) &&
586 isValidBranch(BB, BI, Op1, IsLoopBranch, Context);
587 }
588 }
589
590 if (auto PHI = dyn_cast<PHINode>(Condition)) {
591 auto *Unique = dyn_cast_or_null<ConstantInt>(
592 getUniqueNonErrorValue(PHI, &Context.CurRegion, this));
593 if (Unique && (Unique->isZero() || Unique->isOne()))
594 return true;
595 }
596
597 if (auto Load = dyn_cast<LoadInst>(Condition))
598 if (!IsLoopBranch && Context.CurRegion.contains(Load)) {
599 Context.RequiredILS.insert(Load);
600 return true;
601 }
602
603 // Non constant conditions of branches need to be ICmpInst.
604 if (!isa<ICmpInst>(Condition)) {
605 if (!IsLoopBranch && AllowNonAffineSubRegions &&
606 addOverApproximatedRegion(RI.getRegionFor(&BB), Context))
607 return true;
608 return invalid<ReportInvalidCond>(Context, /*Assert=*/true, BI, &BB);
609 }
610
611 ICmpInst *ICmp = cast<ICmpInst>(Condition);
612
613 // Are both operands of the ICmp affine?
614 if (isa<UndefValue>(ICmp->getOperand(0)) ||
615 isa<UndefValue>(ICmp->getOperand(1)))
616 return invalid<ReportUndefOperand>(Context, /*Assert=*/true, &BB, ICmp);
617
618 Loop *L = LI.getLoopFor(&BB);
619 const SCEV *LHS = SE.getSCEVAtScope(ICmp->getOperand(0), L);
620 const SCEV *RHS = SE.getSCEVAtScope(ICmp->getOperand(1), L);
621
622 LHS = tryForwardThroughPHI(LHS, Context.CurRegion, SE, this);
623 RHS = tryForwardThroughPHI(RHS, Context.CurRegion, SE, this);
624
625 // If unsigned operations are not allowed try to approximate the region.
626 if (ICmp->isUnsigned() && !PollyAllowUnsignedOperations)
627 return !IsLoopBranch && AllowNonAffineSubRegions &&
628 addOverApproximatedRegion(RI.getRegionFor(&BB), Context);
629
630 // Check for invalid usage of different pointers in one expression.
631 if (ICmp->isEquality() && involvesMultiplePtrs(LHS, nullptr, L) &&
632 involvesMultiplePtrs(RHS, nullptr, L))
633 return false;
634
635 // Check for invalid usage of different pointers in a relational comparison.
636 if (ICmp->isRelational() && involvesMultiplePtrs(LHS, RHS, L))
637 return false;
638
639 if (isAffine(LHS, L, Context) && isAffine(RHS, L, Context))
640 return true;
641
642 if (!IsLoopBranch && AllowNonAffineSubRegions &&
643 addOverApproximatedRegion(RI.getRegionFor(&BB), Context))
644 return true;
645
646 if (IsLoopBranch)
647 return false;
648
649 return invalid<ReportNonAffBranch>(Context, /*Assert=*/true, &BB, LHS, RHS,
650 ICmp);
651}
652
653bool ScopDetection::isValidCFG(BasicBlock &BB, bool IsLoopBranch,
654 bool AllowUnreachable,
655 DetectionContext &Context) {
656 Region &CurRegion = Context.CurRegion;
657
658 Instruction *TI = BB.getTerminator();
659
660 if (AllowUnreachable && isa<UnreachableInst>(TI))
661 return true;
662
663 // Return instructions are only valid if the region is the top level region.
664 if (isa<ReturnInst>(TI) && CurRegion.isTopLevelRegion())
665 return true;
666
667 if (isa<UncondBrInst>(TI))
668 return true;
669
670 if (auto *BI = dyn_cast<CondBrInst>(TI)) {
671 Value *Condition = BI->getCondition();
672 if (isa<UndefValue>(Condition))
673 return invalid<ReportUndefCond>(Context, /*Assert=*/true, TI, &BB);
674 return isValidBranch(BB, BI, Condition, IsLoopBranch, Context);
675 }
676
677 if (auto *SI = dyn_cast<SwitchInst>(TI)) {
678 Value *Condition = SI->getCondition();
679 if (isa<UndefValue>(Condition))
680 return invalid<ReportUndefCond>(Context, /*Assert=*/true, TI, &BB);
681 return isValidSwitch(BB, SI, Condition, IsLoopBranch, Context);
682 }
683
684 return invalid<ReportInvalidTerminator>(Context, /*Assert=*/true, &BB);
685}
686
688 DetectionContext &Context) const {
689 if (CI.doesNotReturn())
690 return false;
691
692 if (CI.doesNotAccessMemory())
693 return true;
694
695 if (auto *II = dyn_cast<IntrinsicInst>(&CI))
696 if (isValidIntrinsicInst(*II, Context))
697 return true;
698
699 Function *CalledFunction = CI.getCalledFunction();
700
701 // Indirect calls are not supported.
702 if (CalledFunction == nullptr)
703 return false;
704
705 if (isDebugCall(&CI)) {
706 POLLY_DEBUG(dbgs() << "Allow call to debug function: "
707 << CalledFunction->getName() << '\n');
708 return true;
709 }
710
711 if (AllowModrefCall) {
712 MemoryEffects ME = AA.getMemoryEffects(CalledFunction);
713 if (ME.onlyAccessesArgPointees()) {
714 for (const auto &Arg : CI.args()) {
715 if (!Arg->getType()->isPointerTy())
716 continue;
717
718 // Bail if a pointer argument has a base address not known to
719 // ScalarEvolution. Note that a zero pointer is acceptable.
720 const SCEV *ArgSCEV =
721 SE.getSCEVAtScope(Arg, LI.getLoopFor(CI.getParent()));
722 if (ArgSCEV->isZero())
723 continue;
724
725 auto *BP = dyn_cast<SCEVUnknown>(SE.getPointerBase(ArgSCEV));
726 if (!BP)
727 return false;
728
729 // Implicitly disable delinearization since we have an unknown
730 // accesses with an unknown access function.
731 Context.HasUnknownAccess = true;
732 }
733
734 // Explicitly use addUnknown so we don't put a loop-variant
735 // pointer into the alias set.
736 Context.AST.addUnknown(&CI);
737 return true;
738 }
739
740 if (ME.onlyReadsMemory()) {
741 // Implicitly disable delinearization since we have an unknown
742 // accesses with an unknown access function.
743 Context.HasUnknownAccess = true;
744 // Explicitly use addUnknown so we don't put a loop-variant
745 // pointer into the alias set.
746 Context.AST.addUnknown(&CI);
747 return true;
748 }
749 return false;
750 }
751
752 return false;
753}
754
756 DetectionContext &Context) const {
757 if (isIgnoredIntrinsic(&II))
758 return true;
759
760 // The closest loop surrounding the call instruction.
761 Loop *L = LI.getLoopFor(II.getParent());
762
763 // The access function and base pointer for memory intrinsics.
764 const SCEV *AF;
765 const SCEVUnknown *BP;
766
767 switch (II.getIntrinsicID()) {
768 // Memory intrinsics that can be represented are supported.
769 case Intrinsic::memmove:
770 case Intrinsic::memcpy:
771 AF = SE.getSCEVAtScope(cast<MemTransferInst>(II).getSource(), L);
772 if (!AF->isZero()) {
773 BP = dyn_cast<SCEVUnknown>(SE.getPointerBase(AF));
774 // Bail if the source pointer is not valid.
775 if (!isValidAccess(&II, AF, BP, Context))
776 return false;
777 }
778 [[fallthrough]];
779 case Intrinsic::memset:
780 AF = SE.getSCEVAtScope(cast<MemIntrinsic>(II).getDest(), L);
781 if (!AF->isZero()) {
782 BP = dyn_cast<SCEVUnknown>(SE.getPointerBase(AF));
783 // Bail if the destination pointer is not valid.
784 if (!isValidAccess(&II, AF, BP, Context))
785 return false;
786 }
787
788 // Bail if the length is not affine.
789 if (!isAffine(SE.getSCEVAtScope(cast<MemIntrinsic>(II).getLength(), L), L,
790 Context))
791 return false;
792
793 return true;
794 default:
795 break;
796 }
797
798 return false;
799}
800
801bool ScopDetection::isInvariant(Value &Val, const Region &Reg,
802 DetectionContext &Ctx) const {
803 // A reference to function argument or constant value is invariant.
804 if (isa<Argument>(Val) || isa<Constant>(Val))
805 return true;
806
807 Instruction *I = dyn_cast<Instruction>(&Val);
808 if (!I)
809 return false;
810
811 if (!Reg.contains(I))
812 return true;
813
814 // Loads within the SCoP may read arbitrary values, need to hoist them. If it
815 // is not hoistable, it will be rejected later, but here we assume it is and
816 // that makes the value invariant.
817 if (auto LI = dyn_cast<LoadInst>(I)) {
818 Ctx.RequiredILS.insert(LI);
819 return true;
820 }
821
822 return false;
823}
824
825namespace {
826
827/// Remove smax of smax(0, size) expressions from a SCEV expression and
828/// register the '...' components.
829///
830/// Array access expressions as they are generated by GFortran contain smax(0,
831/// size) expressions that confuse the 'normal' delinearization algorithm.
832/// However, if we extract such expressions before the normal delinearization
833/// takes place they can actually help to identify array size expressions in
834/// Fortran accesses. For the subsequently following delinearization the smax(0,
835/// size) component can be replaced by just 'size'. This is correct as we will
836/// always add and verify the assumption that for all subscript expressions
837/// 'exp' the inequality 0 <= exp < size holds. Hence, we will also verify
838/// that 0 <= size, which means smax(0, size) == size.
839class SCEVRemoveMax final : public SCEVRewriteVisitor<SCEVRemoveMax> {
840public:
841 SCEVRemoveMax(ScalarEvolution &SE, std::vector<const SCEV *> *Terms)
842 : SCEVRewriteVisitor(SE), Terms(Terms) {}
843
844 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
845 std::vector<const SCEV *> *Terms = nullptr) {
846 SCEVRemoveMax Rewriter(SE, Terms);
847 return Rewriter.visit(Scev);
848 }
849
850 const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) {
851 if ((Expr->getNumOperands() == 2) && Expr->getOperand(0)->isZero()) {
852 auto Res = visit(Expr->getOperand(1));
853 if (Terms)
854 (*Terms).push_back(Res);
855 return Res;
856 }
857
858 return Expr;
859 }
860
861private:
862 std::vector<const SCEV *> *Terms;
863};
864} // namespace
865
866SmallVector<const SCEV *, 4>
868 const SCEVUnknown *BasePointer) const {
869 SmallVector<const SCEV *, 4> Terms;
870 for (const auto &Pair : Context.Accesses[BasePointer]) {
871 std::vector<const SCEV *> MaxTerms;
872 SCEVRemoveMax::rewrite(Pair.second, SE, &MaxTerms);
873 if (!MaxTerms.empty()) {
874 for (const SCEV *Max : MaxTerms)
875 Terms.push_back(
876 SE.getTruncateOrSignExtend(Max, Pair.second->getType()));
877 continue;
878 }
879 // In case the outermost expression is a plain add, we check if any of its
880 // terms has the form 4 * %inst * %param * %param ..., aka a term that
881 // contains a product between a parameter and an instruction that is
882 // inside the scop. Such instructions, if allowed at all, are instructions
883 // SCEV can not represent, but Polly is still looking through. As a
884 // result, these instructions can depend on induction variables and are
885 // most likely no array sizes. However, terms that are multiplied with
886 // them are likely candidates for array sizes.
887 if (auto *AF = dyn_cast<SCEVAddExpr>(Pair.second)) {
888 for (auto Op : AF->operands()) {
889 if (auto *AF2 = dyn_cast<SCEVAddRecExpr>(Op))
890 collectParametricTerms(SE, AF2, Terms);
891 if (auto *AF2 = dyn_cast<SCEVMulExpr>(Op)) {
892 SmallVector<SCEVUse, 0> Operands;
893
894 for (const SCEV *MulOp : AF2->operands()) {
895 if (auto *Const = dyn_cast<SCEVConstant>(MulOp))
896 Operands.push_back(Const);
897 if (auto *Unknown = dyn_cast<SCEVUnknown>(MulOp)) {
898 if (auto *Inst = dyn_cast<Instruction>(Unknown->getValue())) {
899 if (!Context.CurRegion.contains(Inst))
900 Operands.push_back(MulOp);
901
902 } else {
903 Operands.push_back(MulOp);
904 }
905 }
906 }
907 if (Operands.size())
908 Terms.push_back(SE.getMulExpr(Operands));
909 }
910 }
911 }
912 if (Terms.empty())
913 collectParametricTerms(SE, Pair.second, Terms);
914 }
915 return Terms;
916}
917
919 SmallVectorImpl<const SCEV *> &Sizes,
920 const SCEVUnknown *BasePointer,
921 Loop *Scope) const {
922 // If no sizes were found, all sizes are trivially valid. We allow this case
923 // to make it possible to pass known-affine accesses to the delinearization to
924 // try to recover some interesting multi-dimensional accesses, but to still
925 // allow the already known to be affine access in case the delinearization
926 // fails. In such situations, the delinearization will just return a Sizes
927 // array of size zero.
928 if (Sizes.size() == 0)
929 return true;
930
931 Value *BaseValue = BasePointer->getValue();
932 Region &CurRegion = Context.CurRegion;
933 for (const SCEV *DelinearizedSize : Sizes) {
934 // Don't pass down the scope to isAfffine; array dimensions must be
935 // invariant across the entire scop.
936 if (!isAffine(DelinearizedSize, nullptr, Context)) {
937 Sizes.clear();
938 break;
939 }
940 if (auto *Unknown = dyn_cast<SCEVUnknown>(DelinearizedSize)) {
941 auto *V = dyn_cast<Value>(Unknown->getValue());
942 if (auto *Load = dyn_cast<LoadInst>(V)) {
943 if (Context.CurRegion.contains(Load) &&
944 isHoistableLoad(Load, CurRegion, LI, SE, DT, Context.RequiredILS))
945 Context.RequiredILS.insert(Load);
946 continue;
947 }
948 }
949 if (hasScalarDepsInsideRegion(DelinearizedSize, &CurRegion, Scope, false,
950 Context.RequiredILS))
952 Context, /*Assert=*/true, DelinearizedSize,
953 Context.Accesses[BasePointer].front().first, BaseValue);
954 }
955
956 // No array shape derived.
957 if (Sizes.empty()) {
958 if (AllowNonAffine)
959 return true;
960
961 for (const auto &Pair : Context.Accesses[BasePointer]) {
962 const Instruction *Insn = Pair.first;
963 const SCEV *AF = Pair.second;
964
965 if (!isAffine(AF, Scope, Context)) {
966 invalid<ReportNonAffineAccess>(Context, /*Assert=*/true, AF, Insn,
967 BaseValue);
968 if (!KeepGoing)
969 return false;
970 }
971 }
972 return false;
973 }
974 return true;
975}
976
977// We first store the resulting memory accesses in TempMemoryAccesses. Only
978// if the access functions for all memory accesses have been successfully
979// delinearized we continue. Otherwise, we either report a failure or, if
980// non-affine accesses are allowed, we drop the information. In case the
981// information is dropped the memory accesses need to be overapproximated
982// when translated to a polyhedral representation.
984 DetectionContext &Context, const SCEVUnknown *BasePointer,
985 std::shared_ptr<ArrayShape> Shape) const {
986 Value *BaseValue = BasePointer->getValue();
987 bool BasePtrHasNonAffine = false;
988 MapInsnToMemAcc TempMemoryAccesses;
989 for (const auto &Pair : Context.Accesses[BasePointer]) {
990 const Instruction *Insn = Pair.first;
991 auto *AF = Pair.second;
992 AF = SCEVRemoveMax::rewrite(AF, SE);
993 bool IsNonAffine = false;
994 TempMemoryAccesses.insert(std::make_pair(Insn, MemAcc(Insn, Shape)));
995 MemAcc *Acc = &TempMemoryAccesses.find(Insn)->second;
996 auto *Scope = LI.getLoopFor(Insn->getParent());
997
998 if (!AF) {
999 if (isAffine(Pair.second, Scope, Context))
1000 Acc->DelinearizedSubscripts.push_back(Pair.second);
1001 else
1002 IsNonAffine = true;
1003 } else {
1004 if (Shape->DelinearizedSizes.size() == 0) {
1005 Acc->DelinearizedSubscripts.push_back(AF);
1006 } else {
1007 llvm::computeAccessFunctions(SE, AF, Acc->DelinearizedSubscripts,
1008 Shape->DelinearizedSizes);
1009 if (Acc->DelinearizedSubscripts.size() == 0)
1010 IsNonAffine = true;
1011 }
1012 for (const SCEV *S : Acc->DelinearizedSubscripts)
1013 if (!isAffine(S, Scope, Context))
1014 IsNonAffine = true;
1015 }
1016
1017 // (Possibly) report non affine access
1018 if (IsNonAffine) {
1019 BasePtrHasNonAffine = true;
1020 if (!AllowNonAffine) {
1021 invalid<ReportNonAffineAccess>(Context, /*Assert=*/true, Pair.second,
1022 Insn, BaseValue);
1023 if (!KeepGoing)
1024 return false;
1025 }
1026 }
1027 }
1028
1029 if (!BasePtrHasNonAffine)
1030 Context.InsnToMemAcc.insert(TempMemoryAccesses.begin(),
1031 TempMemoryAccesses.end());
1032
1033 return true;
1034}
1035
1037 const SCEVUnknown *BasePointer,
1038 Loop *Scope) const {
1039 auto Shape = std::shared_ptr<ArrayShape>(new ArrayShape(BasePointer));
1040
1041 auto Terms = getDelinearizationTerms(Context, BasePointer);
1042
1043 findArrayDimensions(SE, Terms, Shape->DelinearizedSizes,
1044 Context.ElementSize[BasePointer]);
1045
1046 if (!hasValidArraySizes(Context, Shape->DelinearizedSizes, BasePointer,
1047 Scope))
1048 return false;
1049
1050 return computeAccessFunctions(Context, BasePointer, Shape);
1051}
1052
1054 // TODO: If we have an unknown access and other non-affine accesses we do
1055 // not try to delinearize them for now.
1056 if (Context.HasUnknownAccess && !Context.NonAffineAccesses.empty())
1057 return AllowNonAffine;
1058
1059 for (auto &Pair : Context.NonAffineAccesses) {
1060 auto *BasePointer = Pair.first;
1061 auto *Scope = Pair.second;
1062 if (!hasBaseAffineAccesses(Context, BasePointer, Scope)) {
1063 Context.IsInvalid = true;
1064 if (!KeepGoing)
1065 return false;
1066 }
1067 }
1068 return true;
1069}
1070
1071bool ScopDetection::isValidAccess(Instruction *Inst, const SCEV *AF,
1072 const SCEVUnknown *BP,
1073 DetectionContext &Context) const {
1074
1075 if (!BP)
1076 return invalid<ReportNoBasePtr>(Context, /*Assert=*/true, Inst);
1077
1078 auto *BV = BP->getValue();
1079 if (isa<UndefValue>(BV))
1080 return invalid<ReportUndefBasePtr>(Context, /*Assert=*/true, Inst);
1081
1082 // FIXME: Think about allowing IntToPtrInst
1083 if (IntToPtrInst *Inst = dyn_cast<IntToPtrInst>(BV))
1084 return invalid<ReportIntToPtr>(Context, /*Assert=*/true, Inst);
1085
1086 // Check that the base address of the access is invariant in the current
1087 // region.
1088 if (!isInvariant(*BV, Context.CurRegion, Context))
1089 return invalid<ReportVariantBasePtr>(Context, /*Assert=*/true, BV, Inst);
1090
1091 AF = SE.getMinusSCEV(AF, BP);
1092
1093 const SCEV *Size;
1094 if (!isa<MemIntrinsic>(Inst)) {
1095 Size = SE.getElementSize(Inst);
1096 } else {
1097 auto *SizeTy =
1098 SE.getEffectiveSCEVType(PointerType::getUnqual(SE.getContext()));
1099 Size = SE.getConstant(SizeTy, 8);
1100 }
1101
1102 if (Context.ElementSize[BP]) {
1103 if (!AllowDifferentTypes && Context.ElementSize[BP] != Size)
1104 return invalid<ReportDifferentArrayElementSize>(Context, /*Assert=*/true,
1105 Inst, BV);
1106
1107 Context.ElementSize[BP] = SE.getSMinExpr(Size, Context.ElementSize[BP]);
1108 } else {
1109 Context.ElementSize[BP] = Size;
1110 }
1111
1112 bool IsVariantInNonAffineLoop = false;
1113 SetVector<const Loop *> Loops;
1114 findLoops(AF, Loops);
1115 for (const Loop *L : Loops)
1116 if (Context.BoxedLoopsSet.count(L))
1117 IsVariantInNonAffineLoop = true;
1118
1119 auto *Scope = LI.getLoopFor(Inst->getParent());
1120 bool IsAffine = !IsVariantInNonAffineLoop && isAffine(AF, Scope, Context);
1121 // Do not try to delinearize memory intrinsics and force them to be affine.
1122 if (isa<MemIntrinsic>(Inst) && !IsAffine) {
1123 return invalid<ReportNonAffineAccess>(Context, /*Assert=*/true, AF, Inst,
1124 BV);
1125 } else if (PollyDelinearize && !IsVariantInNonAffineLoop) {
1126 Context.Accesses[BP].push_back({Inst, AF});
1127
1128 if (!IsAffine)
1129 Context.NonAffineAccesses.insert(
1130 std::make_pair(BP, LI.getLoopFor(Inst->getParent())));
1131 } else if (!AllowNonAffine && !IsAffine) {
1132 return invalid<ReportNonAffineAccess>(Context, /*Assert=*/true, AF, Inst,
1133 BV);
1134 }
1135
1136 if (IgnoreAliasing)
1137 return true;
1138
1139 // Check if the base pointer of the memory access does alias with
1140 // any other pointer. This cannot be handled at the moment.
1141 AAMDNodes AATags = Inst->getAAMetadata();
1142 AliasSet &AS = Context.AST.getAliasSetFor(
1143 MemoryLocation::getBeforeOrAfter(BP->getValue(), AATags));
1144
1145 if (!AS.isMustAlias()) {
1147 bool CanBuildRunTimeCheck = true;
1148 // The run-time alias check places code that involves the base pointer at
1149 // the beginning of the SCoP. This breaks if the base pointer is defined
1150 // inside the scop. Hence, we can only create a run-time check if we are
1151 // sure the base pointer is not an instruction defined inside the scop.
1152 // However, we can ignore loads that will be hoisted.
1153
1154 auto ASPointers = AS.getPointers();
1155
1156 InvariantLoadsSetTy VariantLS, InvariantLS;
1157 // In order to detect loads which are dependent on other invariant loads
1158 // as invariant, we use fixed-point iteration method here i.e we iterate
1159 // over the alias set for arbitrary number of times until it is safe to
1160 // assume that all the invariant loads have been detected
1161 while (true) {
1162 const unsigned int VariantSize = VariantLS.size(),
1163 InvariantSize = InvariantLS.size();
1164
1165 for (const Value *Ptr : ASPointers) {
1166 Instruction *Inst = dyn_cast<Instruction>(const_cast<Value *>(Ptr));
1167 if (Inst && Context.CurRegion.contains(Inst)) {
1168 auto *Load = dyn_cast<LoadInst>(Inst);
1169 if (Load && InvariantLS.count(Load))
1170 continue;
1171 if (Load && isHoistableLoad(Load, Context.CurRegion, LI, SE, DT,
1172 InvariantLS)) {
1173 if (VariantLS.count(Load))
1174 VariantLS.remove(Load);
1175 Context.RequiredILS.insert(Load);
1176 InvariantLS.insert(Load);
1177 } else {
1178 CanBuildRunTimeCheck = false;
1179 VariantLS.insert(Load);
1180 }
1181 }
1182 }
1183
1184 if (InvariantSize == InvariantLS.size() &&
1185 VariantSize == VariantLS.size())
1186 break;
1187 }
1188
1189 if (CanBuildRunTimeCheck)
1190 return true;
1191 }
1192 return invalid<ReportAlias>(Context, /*Assert=*/true, Inst, AS);
1193 }
1194
1195 return true;
1196}
1197
1199 DetectionContext &Context) const {
1200 Value *Ptr = Inst.getPointerOperand();
1201 Loop *L = LI.getLoopFor(Inst->getParent());
1202 const SCEV *AccessFunction = SE.getSCEVAtScope(Ptr, L);
1203 const SCEVUnknown *BasePointer;
1204
1205 BasePointer = dyn_cast<SCEVUnknown>(SE.getPointerBase(AccessFunction));
1206
1207 return isValidAccess(Inst, AccessFunction, BasePointer, Context);
1208}
1209
1210bool ScopDetection::isCompatibleType(Instruction *Inst, Type *Ty,
1211 DetectionContext &Context) {
1212 if (!Ty)
1213 return false;
1214
1215 if (isa<ScalableVectorType>(Ty))
1216 return invalid<ReportIncompatibleType>(Context, /*Assert=*/true, Inst, Ty);
1217
1218 return true;
1219}
1220
1222 DetectionContext &Context) {
1223 for (auto &Op : Inst.operands()) {
1224 auto *OpInst = dyn_cast<Instruction>(&Op);
1225
1226 if (!OpInst)
1227 continue;
1228
1229 if (!isCompatibleType(&Inst, Op->getType(), Context))
1230 return false;
1231
1232 if (isErrorBlock(*OpInst->getParent(), Context.CurRegion)) {
1233 auto *PHI = dyn_cast<PHINode>(OpInst);
1234 if (PHI) {
1235 for (User *U : PHI->users()) {
1236 auto *UI = dyn_cast<Instruction>(U);
1237 if (!UI || !UI->isTerminator())
1238 return false;
1239 }
1240 } else {
1241 return false;
1242 }
1243 }
1244 }
1245
1246 if (isa<LandingPadInst>(&Inst) || isa<ResumeInst>(&Inst))
1247 return false;
1248
1249 if (!isCompatibleType(&Inst, Inst.getType(), Context))
1250 return false;
1251
1252 // We only check the call instruction but not invoke instruction.
1253 if (CallInst *CI = dyn_cast<CallInst>(&Inst)) {
1254 if (isValidCallInst(*CI, Context))
1255 return true;
1256
1257 return invalid<ReportFuncCall>(Context, /*Assert=*/true, &Inst);
1258 }
1259
1260 if (!Inst.mayReadOrWriteMemory()) {
1261 if (!isa<AllocaInst>(Inst))
1262 return true;
1263
1264 return invalid<ReportAlloca>(Context, /*Assert=*/true, &Inst);
1265 }
1266
1267 // Check the access function.
1268 if (auto MemInst = MemAccInst::dyn_cast(Inst)) {
1269 Context.hasStores |= isa<StoreInst>(MemInst);
1270 Context.hasLoads |= isa<LoadInst>(MemInst);
1271 if (!MemInst.isSimple())
1272 return invalid<ReportNonSimpleMemoryAccess>(Context, /*Assert=*/true,
1273 &Inst);
1274
1275 return isValidMemoryAccess(MemInst, Context);
1276 }
1277
1278 // We do not know this instruction, therefore we assume it is invalid.
1279 return invalid<ReportUnknownInst>(Context, /*Assert=*/true, &Inst);
1280}
1281
1282/// Check whether @p L has exiting blocks.
1283///
1284/// @param L The loop of interest
1285///
1286/// @return True if the loop has exiting blocks, false otherwise.
1287static bool hasExitingBlocks(Loop *L) {
1288 SmallVector<BasicBlock *, 4> ExitingBlocks;
1289 L->getExitingBlocks(ExitingBlocks);
1290 return !ExitingBlocks.empty();
1291}
1292
1294 // FIXME: Yes, this is bad. isValidCFG() may call invalid<Reason>() which
1295 // causes the SCoP to be rejected regardless on whether non-ISL trip counts
1296 // could be used. We currently preserve the legacy behaviour of rejecting
1297 // based on Context.Log.size() added by isValidCFG() or before, regardless on
1298 // whether the ISL trip count can be used or can be used as a non-affine
1299 // region. However, we allow rejections by isValidCFG() that do not result in
1300 // an error log entry.
1301 bool OldIsInvalid = Context.IsInvalid;
1302
1303 // Ensure the loop has valid exiting blocks as well as latches, otherwise we
1304 // need to overapproximate it as a boxed loop.
1305 SmallVector<BasicBlock *, 4> LoopControlBlocks;
1306 L->getExitingBlocks(LoopControlBlocks);
1307 L->getLoopLatches(LoopControlBlocks);
1308 for (BasicBlock *ControlBB : LoopControlBlocks) {
1309 if (!isValidCFG(*ControlBB, true, false, Context)) {
1310 Context.IsInvalid = OldIsInvalid || Context.Log.size();
1311 return false;
1312 }
1313 }
1314
1315 // We can use ISL to compute the trip count of L.
1316 Context.IsInvalid = OldIsInvalid || Context.Log.size();
1317 return true;
1318}
1319
1321 // Loops that contain part but not all of the blocks of a region cannot be
1322 // handled by the schedule generation. Such loop constructs can happen
1323 // because a region can contain BBs that have no path to the exit block
1324 // (Infinite loops, UnreachableInst), but such blocks are never part of a
1325 // loop.
1326 //
1327 // _______________
1328 // | Loop Header | <-----------.
1329 // --------------- |
1330 // | |
1331 // _______________ ______________
1332 // | RegionEntry |-----> | RegionExit |----->
1333 // --------------- --------------
1334 // |
1335 // _______________
1336 // | EndlessLoop | <--.
1337 // --------------- |
1338 // | |
1339 // \------------/
1340 //
1341 // In the example above, the loop (LoopHeader,RegionEntry,RegionExit) is
1342 // neither entirely contained in the region RegionEntry->RegionExit
1343 // (containing RegionEntry,EndlessLoop) nor is the region entirely contained
1344 // in the loop.
1345 // The block EndlessLoop is contained in the region because Region::contains
1346 // tests whether it is not dominated by RegionExit. This is probably to not
1347 // having to query the PostdominatorTree. Instead of an endless loop, a dead
1348 // end can also be formed by an UnreachableInst. This case is already caught
1349 // by isErrorBlock(). We hence only have to reject endless loops here.
1350 if (!hasExitingBlocks(L))
1351 return invalid<ReportLoopHasNoExit>(Context, /*Assert=*/true, L);
1352
1353 // The algorithm for domain construction assumes that loops has only a single
1354 // exit block (and hence corresponds to a subregion). Note that we cannot use
1355 // L->getExitBlock() because it does not check whether all exiting edges point
1356 // to the same BB.
1357 SmallVector<BasicBlock *, 4> ExitBlocks;
1358 L->getExitBlocks(ExitBlocks);
1359 BasicBlock *TheExitBlock = ExitBlocks[0];
1360 for (BasicBlock *ExitBB : ExitBlocks) {
1361 if (TheExitBlock != ExitBB)
1362 return invalid<ReportLoopHasMultipleExits>(Context, /*Assert=*/true, L);
1363 }
1364
1365 if (canUseISLTripCount(L, Context))
1366 return true;
1367
1369 Region *R = RI.getRegionFor(L->getHeader());
1370 while (R != &Context.CurRegion && !R->contains(L))
1371 R = R->getParent();
1372
1373 if (addOverApproximatedRegion(R, Context))
1374 return true;
1375 }
1376
1377 const SCEV *LoopCount = SE.getBackedgeTakenCount(L);
1378 return invalid<ReportLoopBound>(Context, /*Assert=*/true, L, LoopCount);
1379}
1380
1381/// Return the number of loops in @p L (incl. @p L) that have a trip
1382/// count that is not known to be less than @MinProfitableTrips.
1385 unsigned MinProfitableTrips) {
1386 const SCEV *TripCount = SE.getBackedgeTakenCount(L);
1387
1388 int NumLoops = 1;
1389 int MaxLoopDepth = 1;
1390 if (MinProfitableTrips > 0)
1391 if (auto *TripCountC = dyn_cast<SCEVConstant>(TripCount))
1392 if (TripCountC->getType()->getScalarSizeInBits() <= 64)
1393 if (TripCountC->getValue()->getZExtValue() <= MinProfitableTrips)
1394 NumLoops -= 1;
1395
1396 for (auto &SubLoop : *L) {
1397 LoopStats Stats = countBeneficialSubLoops(SubLoop, SE, MinProfitableTrips);
1398 NumLoops += Stats.NumLoops;
1399 MaxLoopDepth = std::max(MaxLoopDepth, Stats.MaxDepth + 1);
1400 }
1401
1402 return {NumLoops, MaxLoopDepth};
1403}
1404
1406ScopDetection::countBeneficialLoops(Region *R, ScalarEvolution &SE,
1407 LoopInfo &LI, unsigned MinProfitableTrips) {
1408 int LoopNum = 0;
1409 int MaxLoopDepth = 0;
1410
1411 auto L = LI.getLoopFor(R->getEntry());
1412
1413 // If L is fully contained in R, move to first loop surrounding R. Otherwise,
1414 // L is either nullptr or already surrounding R.
1415 if (L && R->contains(L)) {
1416 L = R->outermostLoopInRegion(L);
1417 L = L->getParentLoop();
1418 }
1419
1420 auto SubLoops =
1421 L ? L->getSubLoops() : std::vector<Loop *>(LI.begin(), LI.end());
1422
1423 for (auto &SubLoop : SubLoops)
1424 if (R->contains(SubLoop)) {
1425 LoopStats Stats =
1426 countBeneficialSubLoops(SubLoop, SE, MinProfitableTrips);
1427 LoopNum += Stats.NumLoops;
1428 MaxLoopDepth = std::max(MaxLoopDepth, Stats.MaxDepth);
1429 }
1430
1431 return {LoopNum, MaxLoopDepth};
1432}
1433
1434static bool isErrorBlockImpl(BasicBlock &BB, const Region &R, LoopInfo &LI,
1435 const DominatorTree &DT) {
1436 if (isa<UnreachableInst>(BB.getTerminator()))
1437 return true;
1438
1439 if (LI.isLoopHeader(&BB))
1440 return false;
1441
1442 // Don't consider something outside the SCoP as error block. It will precede
1443 // the code versioning runtime check.
1444 if (!R.contains(&BB))
1445 return false;
1446
1447 // Basic blocks that are always executed are not considered error blocks,
1448 // as their execution can not be a rare event.
1449 bool DominatesAllPredecessors = true;
1450 if (R.isTopLevelRegion()) {
1451 for (BasicBlock &I : *R.getEntry()->getParent()) {
1452 if (isa<ReturnInst>(I.getTerminator()) && !DT.dominates(&BB, &I)) {
1453 DominatesAllPredecessors = false;
1454 break;
1455 }
1456 }
1457 } else {
1458 for (auto Pred : predecessors(R.getExit())) {
1459 if (R.contains(Pred) && !DT.dominates(&BB, Pred)) {
1460 DominatesAllPredecessors = false;
1461 break;
1462 }
1463 }
1464 }
1465
1466 if (DominatesAllPredecessors)
1467 return false;
1468
1469 for (Instruction &Inst : BB)
1470 if (CallInst *CI = dyn_cast<CallInst>(&Inst)) {
1471 if (isDebugCall(CI))
1472 continue;
1473
1474 if (isIgnoredIntrinsic(CI))
1475 continue;
1476
1477 // memset, memcpy and memmove are modeled intrinsics.
1478 if (isa<MemSetInst>(CI) || isa<MemTransferInst>(CI))
1479 continue;
1480
1481 if (!CI->doesNotAccessMemory())
1482 return true;
1483 if (CI->doesNotReturn())
1484 return true;
1485 }
1486
1487 return false;
1488}
1489
1490bool ScopDetection::isErrorBlock(llvm::BasicBlock &BB, const llvm::Region &R) {
1492 return false;
1493
1494 auto It = ErrorBlockCache.insert({std::make_pair(&BB, &R), false});
1495 if (!It.second)
1496 return It.first->getSecond();
1497
1498 bool Result = isErrorBlockImpl(BB, R, LI, DT);
1499 It.first->second = Result;
1500 return Result;
1501}
1502
1504 // Initial no valid region was found (greater than R)
1505 std::unique_ptr<Region> LastValidRegion;
1506 auto ExpandedRegion = std::unique_ptr<Region>(R.getExpandedRegion());
1507
1508 POLLY_DEBUG(dbgs() << "\tExpanding " << R.getNameStr() << "\n");
1509
1510 while (ExpandedRegion) {
1511 BBPair P = getBBPairForRegion(ExpandedRegion.get());
1512 std::unique_ptr<DetectionContext> &Entry = DetectionContextMap[P];
1513 Entry = std::make_unique<DetectionContext>(*ExpandedRegion, AA,
1514 /*Verifying=*/false);
1515 DetectionContext &Context = *Entry;
1516
1517 POLLY_DEBUG(dbgs() << "\t\tTrying " << ExpandedRegion->getNameStr()
1518 << "\n");
1519 // Only expand when we did not collect errors.
1520
1521 if (!Context.Log.hasErrors()) {
1522 // If the exit is valid check all blocks
1523 // - if true, a valid region was found => store it + keep expanding
1524 // - if false, .tbd. => stop (should this really end the loop?)
1525 if (!allBlocksValid(Context) || Context.Log.hasErrors()) {
1526 removeCachedResults(*ExpandedRegion);
1527 DetectionContextMap.erase(P);
1528 break;
1529 }
1530
1531 // Store this region, because it is the greatest valid (encountered so
1532 // far).
1533 if (LastValidRegion) {
1534 removeCachedResults(*LastValidRegion);
1535 DetectionContextMap.erase(P);
1536 }
1537 LastValidRegion = std::move(ExpandedRegion);
1538
1539 // Create and test the next greater region (if any)
1540 ExpandedRegion =
1541 std::unique_ptr<Region>(LastValidRegion->getExpandedRegion());
1542
1543 } else {
1544 // Create and test the next greater region (if any)
1545 removeCachedResults(*ExpandedRegion);
1546 DetectionContextMap.erase(P);
1547 ExpandedRegion =
1548 std::unique_ptr<Region>(ExpandedRegion->getExpandedRegion());
1549 }
1550 }
1551
1552 POLLY_DEBUG({
1553 if (LastValidRegion)
1554 dbgs() << "\tto " << LastValidRegion->getNameStr() << "\n";
1555 else
1556 dbgs() << "\tExpanding " << R.getNameStr() << " failed\n";
1557 });
1558
1559 return LastValidRegion.release();
1560}
1561
1562static bool regionWithoutLoops(Region &R, LoopInfo &LI) {
1563 for (const BasicBlock *BB : R.blocks())
1564 if (R.contains(LI.getLoopFor(BB)))
1565 return false;
1566
1567 return true;
1568}
1569
1571 for (auto &SubRegion : R) {
1572 if (ValidRegions.count(SubRegion.get())) {
1573 removeCachedResults(*SubRegion);
1574 } else
1576 }
1577}
1578
1580 ValidRegions.remove(&R);
1581}
1582
1584 std::unique_ptr<DetectionContext> &Entry =
1586 Entry = std::make_unique<DetectionContext>(R, AA, /*Verifying=*/false);
1587 DetectionContext &Context = *Entry;
1588
1589 bool DidBailout = true;
1591 invalid<ReportUnprofitable>(Context, /*Assert=*/true, &R);
1592 else
1593 DidBailout = !isValidRegion(Context);
1594
1595 (void)DidBailout;
1596 if (KeepGoing) {
1597 assert((!DidBailout || Context.IsInvalid) &&
1598 "With -polly-detect-keep-going, it is sufficient that if "
1599 "isValidRegion short-circuited, that SCoP is invalid");
1600 } else {
1601 assert(DidBailout == Context.IsInvalid &&
1602 "isValidRegion must short-circuit iff the ScoP is invalid");
1603 }
1604
1605 if (Context.IsInvalid) {
1607 } else {
1608 ValidRegions.insert(&R);
1609 return;
1610 }
1611
1612 for (auto &SubRegion : R)
1613 findScops(*SubRegion);
1614
1615 // Try to expand regions.
1616 //
1617 // As the region tree normally only contains canonical regions, non canonical
1618 // regions that form a Scop are not found. Therefore, those non canonical
1619 // regions are checked by expanding the canonical ones.
1620
1621 std::vector<Region *> ToExpand;
1622
1623 for (auto &SubRegion : R)
1624 ToExpand.push_back(SubRegion.get());
1625
1626 for (Region *CurrentRegion : ToExpand) {
1627 // Skip invalid regions. Regions may become invalid, if they are element of
1628 // an already expanded region.
1629 if (!ValidRegions.count(CurrentRegion))
1630 continue;
1631
1632 // Skip regions that had errors.
1633 bool HadErrors = lookupRejectionLog(CurrentRegion)->hasErrors();
1634 if (HadErrors)
1635 continue;
1636
1637 Region *ExpandedR = expandRegion(*CurrentRegion);
1638
1639 if (!ExpandedR)
1640 continue;
1641
1642 R.addSubRegion(ExpandedR, true);
1643 ValidRegions.insert(ExpandedR);
1644 removeCachedResults(*CurrentRegion);
1646 }
1647}
1648
1650 Region &CurRegion = Context.CurRegion;
1651
1652 for (const BasicBlock *BB : CurRegion.blocks()) {
1653 Loop *L = LI.getLoopFor(BB);
1654 if (L && L->getHeader() == BB) {
1655 if (CurRegion.contains(L)) {
1656 if (!isValidLoop(L, Context)) {
1657 Context.IsInvalid = true;
1658 if (!KeepGoing)
1659 return false;
1660 }
1661 } else {
1662 SmallVector<BasicBlock *, 1> Latches;
1663 L->getLoopLatches(Latches);
1664 for (BasicBlock *Latch : Latches)
1665 if (CurRegion.contains(Latch))
1666 return invalid<ReportLoopOnlySomeLatches>(Context, /*Assert=*/true,
1667 L);
1668 }
1669 }
1670 }
1671
1672 for (BasicBlock *BB : CurRegion.blocks()) {
1673 bool IsErrorBlock = isErrorBlock(*BB, CurRegion);
1674
1675 // Also check exception blocks (and possibly register them as non-affine
1676 // regions). Even though exception blocks are not modeled, we use them
1677 // to forward-propagate domain constraints during ScopInfo construction.
1678 if (!isValidCFG(*BB, false, IsErrorBlock, Context) && !KeepGoing)
1679 return false;
1680
1681 if (IsErrorBlock)
1682 continue;
1683
1684 for (BasicBlock::iterator I = BB->begin(), E = --BB->end(); I != E; ++I)
1685 if (!isValidInstruction(*I, Context)) {
1686 Context.IsInvalid = true;
1687 if (!KeepGoing)
1688 return false;
1689 }
1690 }
1691
1692 if (!hasAffineMemoryAccesses(Context))
1693 return false;
1694
1695 return true;
1696}
1697
1699 int NumLoops) const {
1700 int InstCount = 0;
1701
1702 if (NumLoops == 0)
1703 return false;
1704
1705 for (auto *BB : Context.CurRegion.blocks())
1706 if (Context.CurRegion.contains(LI.getLoopFor(BB)))
1707 InstCount += BB->size();
1708
1709 InstCount = InstCount / NumLoops;
1710
1711 return InstCount >= ProfitabilityMinPerLoopInstructions;
1712}
1713
1715 DetectionContext &Context) const {
1716 for (auto *BB : Context.CurRegion.blocks()) {
1717 auto *L = LI.getLoopFor(BB);
1718 if (!L)
1719 continue;
1720 if (!Context.CurRegion.contains(L))
1721 continue;
1722 if (Context.BoxedLoopsSet.count(L))
1723 continue;
1724 unsigned StmtsWithStoresInLoops = 0;
1725 for (auto *LBB : L->blocks()) {
1726 bool MemStore = false;
1727 for (auto &I : *LBB)
1728 MemStore |= isa<StoreInst>(&I);
1729 StmtsWithStoresInLoops += MemStore;
1730 }
1731 return (StmtsWithStoresInLoops > 1);
1732 }
1733 return false;
1734}
1735
1737 Region &CurRegion = Context.CurRegion;
1738
1740 return true;
1741
1742 // We can probably not do a lot on scops that only write or only read
1743 // data.
1744 if (!Context.hasStores || !Context.hasLoads)
1745 return invalid<ReportUnprofitable>(Context, /*Assert=*/true, &CurRegion);
1746
1747 int NumLoops =
1749 int NumAffineLoops = NumLoops - Context.BoxedLoopsSet.size();
1750
1751 // Scops with at least two loops may allow either loop fusion or tiling and
1752 // are consequently interesting to look at.
1753 if (NumAffineLoops >= 2)
1754 return true;
1755
1756 // A loop with multiple non-trivial blocks might be amendable to distribution.
1757 if (NumAffineLoops == 1 && hasPossiblyDistributableLoop(Context))
1758 return true;
1759
1760 // Scops that contain a loop with a non-trivial amount of computation per
1761 // loop-iteration are interesting as we may be able to parallelize such
1762 // loops. Individual loops that have only a small amount of computation
1763 // per-iteration are performance-wise very fragile as any change to the
1764 // loop induction variables may affect performance. To not cause spurious
1765 // performance regressions, we do not consider such loops.
1766 if (NumAffineLoops == 1 && hasSufficientCompute(Context, NumLoops))
1767 return true;
1768
1769 return invalid<ReportUnprofitable>(Context, /*Assert=*/true, &CurRegion);
1770}
1771
1773 Region &CurRegion = Context.CurRegion;
1774
1775 POLLY_DEBUG(dbgs() << "Checking region: " << CurRegion.getNameStr()
1776 << "\n\t");
1777
1778 if (!PollyAllowFullFunction && CurRegion.isTopLevelRegion()) {
1779 POLLY_DEBUG(dbgs() << "Top level region is invalid\n");
1780 Context.IsInvalid = true;
1781 return false;
1782 }
1783
1784 DebugLoc DbgLoc;
1785 if (CurRegion.getExit() &&
1786 isa<UnreachableInst>(CurRegion.getExit()->getTerminator())) {
1787 POLLY_DEBUG(dbgs() << "Unreachable in exit\n");
1788 return invalid<ReportUnreachableInExit>(Context, /*Assert=*/true,
1789 CurRegion.getExit(), DbgLoc);
1790 }
1791
1792 if (!OnlyRegion.empty() &&
1793 !CurRegion.getEntry()->getName().count(OnlyRegion)) {
1794 POLLY_DEBUG({
1795 dbgs() << "Region entry does not match -polly-only-region";
1796 dbgs() << "\n";
1797 });
1798 Context.IsInvalid = true;
1799 return false;
1800 }
1801
1802 for (BasicBlock *Pred : predecessors(CurRegion.getEntry())) {
1803 Instruction *PredTerm = Pred->getTerminator();
1804 if (isa<IndirectBrInst>(PredTerm) || isa<CallBrInst>(PredTerm))
1806 Context, /*Assert=*/true, PredTerm, PredTerm->getDebugLoc());
1807 }
1808
1809 // SCoP cannot contain the entry block of the function, because we need
1810 // to insert alloca instruction there when translate scalar to array.
1812 CurRegion.getEntry() ==
1813 &(CurRegion.getEntry()->getParent()->getEntryBlock()))
1814 return invalid<ReportEntry>(Context, /*Assert=*/true, CurRegion.getEntry());
1815
1816 if (!allBlocksValid(Context)) {
1817 // TODO: Every failure condition within allBlocksValid should call
1818 // invalid<Reason>(). Otherwise we reject SCoPs without giving feedback to
1819 // the user.
1820 Context.IsInvalid = true;
1821 return false;
1822 }
1823
1824 if (!isReducibleRegion(CurRegion, DbgLoc))
1825 return invalid<ReportIrreducibleRegion>(Context, /*Assert=*/true,
1826 &CurRegion, DbgLoc);
1827
1828 POLLY_DEBUG(dbgs() << "OK\n");
1829 return true;
1830}
1831
1833 F->addFnAttr(PollySkipFnAttr);
1834}
1835
1837 return !F.hasFnAttribute(PollySkipFnAttr);
1838}
1839
1841 for (const Region *R : *this) {
1842 unsigned LineEntry, LineExit;
1843 std::string FileName;
1844
1845 getDebugLocation(R, LineEntry, LineExit, FileName);
1846 DiagnosticScopFound Diagnostic(F, FileName, LineEntry, LineExit);
1847 F.getContext().diagnose(Diagnostic);
1848 }
1849}
1850
1851void ScopDetection::emitMissedRemarks(const Function &F) {
1852 for (auto &DIt : DetectionContextMap) {
1853 DetectionContext &DC = *DIt.getSecond();
1854 if (DC.Log.hasErrors())
1855 emitRejectionRemarks(DIt.getFirst(), DC.Log, ORE);
1856 }
1857}
1858
1859bool ScopDetection::isReducibleRegion(Region &R, DebugLoc &DbgLoc) const {
1860 /// Enum for coloring BBs in Region.
1861 ///
1862 /// WHITE - Unvisited BB in DFS walk.
1863 /// GREY - BBs which are currently on the DFS stack for processing.
1864 /// BLACK - Visited and completely processed BB.
1865 enum Color { WHITE, GREY, BLACK };
1866
1867 BasicBlock *REntry = R.getEntry();
1868 BasicBlock *RExit = R.getExit();
1869 // Map to match the color of a BasicBlock during the DFS walk.
1870 DenseMap<const BasicBlock *, Color> BBColorMap;
1871 // Stack keeping track of current BB and index of next child to be processed.
1872 std::stack<std::pair<BasicBlock *, unsigned>> DFSStack;
1873
1874 unsigned AdjacentBlockIndex = 0;
1875 BasicBlock *CurrBB, *SuccBB;
1876 CurrBB = REntry;
1877
1878 // Initialize the map for all BB with WHITE color.
1879 for (auto *BB : R.blocks())
1880 BBColorMap[BB] = WHITE;
1881
1882 // Process the entry block of the Region.
1883 BBColorMap[CurrBB] = GREY;
1884 DFSStack.push(std::make_pair(CurrBB, 0));
1885
1886 while (!DFSStack.empty()) {
1887 // Get next BB on stack to be processed.
1888 CurrBB = DFSStack.top().first;
1889 AdjacentBlockIndex = DFSStack.top().second;
1890 DFSStack.pop();
1891
1892 // Loop to iterate over the successors of current BB.
1893 const Instruction *TInst = CurrBB->getTerminator();
1894 unsigned NSucc = TInst->getNumSuccessors();
1895 for (unsigned I = AdjacentBlockIndex; I < NSucc;
1896 ++I, ++AdjacentBlockIndex) {
1897 SuccBB = TInst->getSuccessor(I);
1898
1899 // Checks for region exit block and self-loops in BB.
1900 if (SuccBB == RExit || SuccBB == CurrBB)
1901 continue;
1902
1903 // WHITE indicates an unvisited BB in DFS walk.
1904 if (BBColorMap[SuccBB] == WHITE) {
1905 // Push the current BB and the index of the next child to be visited.
1906 DFSStack.push(std::make_pair(CurrBB, I + 1));
1907 // Push the next BB to be processed.
1908 DFSStack.push(std::make_pair(SuccBB, 0));
1909 // First time the BB is being processed.
1910 BBColorMap[SuccBB] = GREY;
1911 break;
1912 } else if (BBColorMap[SuccBB] == GREY) {
1913 // GREY indicates a loop in the control flow.
1914 // If the destination dominates the source, it is a natural loop
1915 // else, an irreducible control flow in the region is detected.
1916 if (!DT.dominates(SuccBB, CurrBB)) {
1917 // Get debug info of instruction which causes irregular control flow.
1918 DbgLoc = TInst->getDebugLoc();
1919 return false;
1920 }
1921 }
1922 }
1923
1924 // If all children of current BB have been processed,
1925 // then mark that BB as fully processed.
1926 if (AdjacentBlockIndex == NSucc)
1927 BBColorMap[CurrBB] = BLACK;
1928 }
1929
1930 return true;
1931}
1932
1934 bool OnlyProfitable) {
1935 if (!OnlyProfitable) {
1936 NumLoopsInScop += Stats.NumLoops;
1937 MaxNumLoopsInScop =
1938 std::max(MaxNumLoopsInScop.getValue(), (uint64_t)Stats.NumLoops);
1939 if (Stats.MaxDepth == 0)
1940 NumScopsDepthZero++;
1941 else if (Stats.MaxDepth == 1)
1942 NumScopsDepthOne++;
1943 else if (Stats.MaxDepth == 2)
1944 NumScopsDepthTwo++;
1945 else if (Stats.MaxDepth == 3)
1946 NumScopsDepthThree++;
1947 else if (Stats.MaxDepth == 4)
1948 NumScopsDepthFour++;
1949 else if (Stats.MaxDepth == 5)
1950 NumScopsDepthFive++;
1951 else
1952 NumScopsDepthLarger++;
1953 } else {
1954 NumLoopsInProfScop += Stats.NumLoops;
1955 MaxNumLoopsInProfScop =
1956 std::max(MaxNumLoopsInProfScop.getValue(), (uint64_t)Stats.NumLoops);
1957 if (Stats.MaxDepth == 0)
1958 NumProfScopsDepthZero++;
1959 else if (Stats.MaxDepth == 1)
1960 NumProfScopsDepthOne++;
1961 else if (Stats.MaxDepth == 2)
1962 NumProfScopsDepthTwo++;
1963 else if (Stats.MaxDepth == 3)
1964 NumProfScopsDepthThree++;
1965 else if (Stats.MaxDepth == 4)
1966 NumProfScopsDepthFour++;
1967 else if (Stats.MaxDepth == 5)
1968 NumProfScopsDepthFive++;
1969 else
1970 NumProfScopsDepthLarger++;
1971 }
1972}
1973
1976 auto DCMIt = DetectionContextMap.find(getBBPairForRegion(R));
1977 if (DCMIt == DetectionContextMap.end())
1978 return nullptr;
1979 return DCMIt->second.get();
1980}
1981
1982const RejectLog *ScopDetection::lookupRejectionLog(const Region *R) const {
1984 return DC ? &DC->Log : nullptr;
1985}
1986
1987void ScopDetection::verifyRegion(const Region &R) {
1988 assert(isMaxRegionInScop(R) && "Expect R is a valid region.");
1989
1990 DetectionContext Context(const_cast<Region &>(R), AA, true /*verifying*/);
1991 isValidRegion(Context);
1992}
1993
1995 if (!VerifyScops)
1996 return;
1997
1998 for (const Region *R : ValidRegions)
1999 verifyRegion(*R);
2000}
2001
2003 // Disable runtime alias checks if we ignore aliasing all together.
2004 if (IgnoreAliasing)
2006}
2007
2008AnalysisKey ScopAnalysis::Key;
2009
2010ScopDetection ScopAnalysis::run(Function &F, FunctionAnalysisManager &FAM) {
2011 auto &LI = FAM.getResult<LoopAnalysis>(F);
2012 auto &RI = FAM.getResult<RegionInfoAnalysis>(F);
2013 auto &AA = FAM.getResult<AAManager>(F);
2014 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
2015 auto &DT = FAM.getResult<DominatorTreeAnalysis>(F);
2016 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
2017
2018 ScopDetection Result(DT, SE, LI, RI, AA, ORE);
2019 Result.detect(F);
2020 return Result;
2021}
2022
2023PreservedAnalyses ScopAnalysisPrinterPass::run(Function &F,
2024 FunctionAnalysisManager &FAM) {
2025 OS << "Detected Scops in Function " << F.getName() << "\n";
2026 auto &SD = FAM.getResult<ScopAnalysis>(F);
2027 for (const Region *R : SD.ValidRegions)
2028 OS << "Valid Region for Scop: " << R->getNameStr() << '\n';
2029
2030 OS << "\n";
2031 return PreservedAnalyses::all();
2032}
S1()
static cl::opt< bool > Verify("polly-codegen-verify", cl::desc("Verify the function generated by Polly"), cl::Hidden, cl::cat(PollyCategory))
llvm::cl::OptionCategory PollyCategory
#define POLLY_DEBUG(X)
Definition PollyDebug.h:23
static const unsigned MIN_LOOP_TRIP_COUNT
The minimal trip count under which loops are considered unprofitable.
static cl::opt< bool, true > XPollyProcessUnprofitable("polly-process-unprofitable", cl::desc("Process scops that are unlikely to benefit from Polly optimizations."), cl::location(PollyProcessUnprofitable), cl::cat(PollyCategory))
static cl::opt< bool > AllowDifferentTypes("polly-allow-differing-element-types", cl::desc("Allow different element types for array accesses"), cl::Hidden, cl::init(true), cl::cat(PollyCategory))
static cl::opt< bool > AllowNonAffine("polly-allow-nonaffine", cl::desc("Allow non affine access functions in arrays"), cl::Hidden, cl::cat(PollyCategory))
STATISTIC(NumScopRegions, "Number of scops")
static cl::list< std::string > OnlyFunctions("polly-only-func", cl::desc("Only run on functions that match a regex. " "Multiple regexes can be comma separated. " "Scop detection will run on all functions that match " "ANY of the regexes provided."), cl::CommaSeparated, cl::cat(PollyCategory))
static cl::opt< bool > VerifyScops("polly-detect-verify", cl::desc("Verify the detected SCoPs after each transformation"), cl::Hidden, cl::cat(PollyCategory))
static bool hasExitingBlocks(Loop *L)
Check whether L has exiting blocks.
static cl::opt< std::string > OnlyRegion("polly-only-region", cl::desc("Only run on certain regions (The provided identifier must " "appear in the name of the region's entry block"), cl::value_desc("identifier"), cl::ValueRequired, cl::init(""), cl::cat(PollyCategory))
static bool regionWithoutLoops(Region &R, LoopInfo &LI)
static cl::opt< bool > KeepGoing("polly-detect-keep-going", cl::desc("Do not fail on the first error."), cl::Hidden, cl::cat(PollyCategory))
static cl::opt< int > ProfitabilityMinPerLoopInstructions("polly-detect-profitability-min-per-loop-insts", cl::desc("The minimal number of per-loop instructions before a single loop " "region is considered profitable"), cl::Hidden, cl::ValueRequired, cl::init(100000000), cl::cat(PollyCategory))
static cl::opt< bool, true > TrackFailures("polly-detect-track-failures", cl::desc("Track failure strings in detecting scop regions"), cl::location(PollyTrackFailures), cl::Hidden, cl::init(true), cl::cat(PollyCategory))
static bool doesStringMatchAnyRegex(StringRef Str, const cl::list< std::string > &RegexList)
Check if a string matches any regex in a list of regexes.
static cl::opt< bool > PollyAllowErrorBlocks("polly-allow-error-blocks", cl::desc("Allow to speculate on the execution of 'error blocks'."), cl::Hidden, cl::init(true), cl::cat(PollyCategory))
static cl::list< std::string > IgnoredFunctions("polly-ignore-func", cl::desc("Ignore functions that match a regex. " "Multiple regexes can be comma separated. " "Scop detection will ignore all functions that match " "ANY of the regexes provided."), cl::CommaSeparated, cl::cat(PollyCategory))
static cl::opt< bool > ReportLevel("polly-report", cl::desc("Print information about the activities of Polly"), cl::cat(PollyCategory))
static bool isErrorBlockImpl(BasicBlock &BB, const Region &R, LoopInfo &LI, const DominatorTree &DT)
static cl::opt< bool, true > PollyDelinearizeX("polly-delinearize", cl::desc("Delinearize array access functions"), cl::location(PollyDelinearize), cl::Hidden, cl::init(true), cl::cat(PollyCategory))
static cl::opt< bool, true > XPollyAllowUnsignedOperations("polly-allow-unsigned-operations", cl::desc("Allow unsigned operations such as comparisons or zero-extends."), cl::location(PollyAllowUnsignedOperations), cl::Hidden, cl::init(true), cl::cat(PollyCategory))
static void updateLoopCountStatistic(ScopDetection::LoopStats Stats, bool OnlyProfitable)
static cl::opt< bool > AllowNonAffineSubRegions("polly-allow-nonaffine-branches", cl::desc("Allow non affine conditions for branches"), cl::Hidden, cl::init(true), cl::cat(PollyCategory))
static cl::opt< bool, true > XAllowFullFunction("polly-detect-full-functions", cl::desc("Allow the detection of full functions"), cl::location(polly::PollyAllowFullFunction), cl::init(false), cl::cat(PollyCategory))
static cl::opt< bool, true > XPollyInvariantLoadHoisting("polly-invariant-load-hoisting", cl::desc("Hoist invariant loads."), cl::location(PollyInvariantLoadHoisting), cl::Hidden, cl::cat(PollyCategory))
static cl::opt< bool > AllowNonAffineSubLoops("polly-allow-nonaffine-loops", cl::desc("Allow non affine conditions for loops"), cl::Hidden, cl::cat(PollyCategory))
static cl::opt< bool, true > XPollyUseRuntimeAliasChecks("polly-use-runtime-alias-checks", cl::desc("Use runtime alias checks to resolve possible aliasing."), cl::location(PollyUseRuntimeAliasChecks), cl::Hidden, cl::init(true), cl::cat(PollyCategory))
static cl::opt< bool > IgnoreAliasing("polly-ignore-aliasing", cl::desc("Ignore possible aliasing of the array bases"), cl::Hidden, cl::cat(PollyCategory))
static cl::opt< bool > AllowModrefCall("polly-allow-modref-calls", cl::desc("Allow functions with known modref behavior"), cl::Hidden, cl::cat(PollyCategory))
Utility proxy to wrap the common members of LoadInst and StoreInst.
Definition ScopHelper.h:141
static MemAccInst dyn_cast(llvm::Value &V)
Definition ScopHelper.h:179
llvm::Value * getPointerOperand() const
Definition ScopHelper.h:249
Stores all errors that occurred during the detection.
void report(RejectReasonPtr Reject)
bool hasErrors() const
Returns true, if we store at least one error.
Base class of all reject reasons found during Scop detection.
virtual std::string getMessage() const =0
Generate a reasonable diagnostic message describing this error.
Pass to detect the maximal static control parts (Scops) of a function.
static void markFunctionAsInvalid(Function *F)
Mark the function as invalid so we will not extract any scop from the function.
bool addOverApproximatedRegion(Region *AR, DetectionContext &Context) const
Add the region AR as over approximated sub-region in Context.
bool isValidAccess(Instruction *Inst, const SCEV *AF, const SCEVUnknown *BP, DetectionContext &Context) const
Check if the memory access caused by Inst is valid.
bool onlyValidRequiredInvariantLoads(InvariantLoadsSetTy &RequiredILS, DetectionContext &Context) const
Check if the given loads could be invariant and can be hoisted.
bool isInvariant(Value &Val, const Region &Reg, DetectionContext &Ctx) const
Check if a value is invariant in the region Reg.
bool isReducibleRegion(Region &R, DebugLoc &DbgLoc) const
Check if a region is reducible or not.
bool computeAccessFunctions(DetectionContext &Context, const SCEVUnknown *BasePointer, std::shared_ptr< ArrayShape > Shape) const
Derive access functions for a given base pointer.
DetectionContext * getDetectionContext(const Region *R) const
Return the detection context for R, nullptr if R was invalid.
void removeCachedResultsRecursively(const Region &R)
Remove cached results for the children of R recursively.
bool hasSufficientCompute(DetectionContext &Context, int NumAffineLoops) const
Check if a region has sufficient compute instructions.
bool isProfitableRegion(DetectionContext &Context) const
Check if a region is profitable to optimize.
void emitMissedRemarks(const Function &F)
Emit rejection remarks for all rejected regions.
bool isValidLoop(Loop *L, DetectionContext &Context)
Is a loop valid with respect to a given region.
static ScopDetection::LoopStats countBeneficialLoops(Region *R, ScalarEvolution &SE, LoopInfo &LI, unsigned MinProfitableTrips)
Count the number of loops and the maximal loop depth in R.
const RejectLog * lookupRejectionLog(const Region *R) const
Return the set of rejection causes for R.
bool involvesMultiplePtrs(const SCEV *S0, const SCEV *S1, Loop *Scope) const
Check if S0 and S1 do contain multiple possibly aliasing pointers.
bool isValidSwitch(BasicBlock &BB, SwitchInst *SI, Value *Condition, bool IsLoopBranch, DetectionContext &Context) const
Check if the switch SI with condition Condition is valid.
bool isValidRegion(DetectionContext &Context)
Check if a region is a Scop.
Region * expandRegion(Region &R)
Try to expand the region R.
const DominatorTree & DT
Analyses used.
bool hasBaseAffineAccesses(DetectionContext &Context, const SCEVUnknown *BasePointer, Loop *Scope) const
Check if all accesses to a given BasePointer are affine.
void detect(Function &F)
ScalarEvolution & SE
OptimizationRemarkEmitter & ORE
OptimizationRemarkEmitter object used to emit diagnostic remarks.
bool hasAffineMemoryAccesses(DetectionContext &Context) const
Delinearize all non affine memory accesses and return false when there exists a non affine memory acc...
bool isValidMemoryAccess(MemAccInst Inst, DetectionContext &Context) const
Check if a memory access can be part of a Scop.
bool isValidCFG(BasicBlock &BB, bool IsLoopBranch, bool AllowUnreachable, DetectionContext &Context)
Check if the control flow in a basic block is valid.
void printLocations(Function &F)
Print the locations of all detected scops.
bool hasValidArraySizes(DetectionContext &Context, SmallVectorImpl< const SCEV * > &Sizes, const SCEVUnknown *BasePointer, Loop *Scope) const
Check if the dimension size of a delinearized array is valid.
DenseMap< std::tuple< const BasicBlock *, const Region * >, bool > ErrorBlockCache
Cache for the isErrorBlock function.
void removeCachedResults(const Region &R)
Remove cached results for R.
ScopDetection(const DominatorTree &DT, ScalarEvolution &SE, LoopInfo &LI, RegionInfo &RI, AAResults &AA, OptimizationRemarkEmitter &ORE)
bool hasPossiblyDistributableLoop(DetectionContext &Context) const
Check if the unique affine loop might be amendable to distribution.
bool isValidBranch(BasicBlock &BB, CondBrInst *BI, Value *Condition, bool IsLoopBranch, DetectionContext &Context)
Check if the branch BI with condition Condition is valid.
void verifyAnalysis()
Verify if all valid Regions in this Function are still valid after some transformations.
SmallVector< const SCEV *, 4 > getDelinearizationTerms(DetectionContext &Context, const SCEVUnknown *BasePointer) const
Find for a given base pointer terms that hint towards dimension sizes of a multi-dimensional array.
bool isValidInstruction(Instruction &Inst, DetectionContext &Context)
Check if an instruction can be part of a Scop.
bool isAffine(const SCEV *S, Loop *Scope, DetectionContext &Context) const
Check if the SCEV S is affine in the current Context.
DetectionContextMapTy DetectionContextMap
bool allBlocksValid(DetectionContext &Context)
Check if all basic block in the region are valid.
void findScops(Region &R)
Find the Scops in this region tree.
bool isValidIntrinsicInst(IntrinsicInst &II, DetectionContext &Context) const
Check if an intrinsic call can be part of a Scop.
std::string regionIsInvalidBecause(const Region *R) const
Get a message why a region is invalid.
bool isMaxRegionInScop(const Region &R, bool Verify=true)
Is the region is the maximum region of a Scop?
bool isValidCallInst(CallInst &CI, DetectionContext &Context) const
Check if a call instruction can be part of a Scop.
void verifyRegion(const Region &R)
Verify if R is still a valid part of Scop after some transformations.
static bool isValidFunction(Function &F)
Check if the function F is marked as invalid.
bool isErrorBlock(llvm::BasicBlock &BB, const llvm::Region &R)
Check if the block is a error block.
bool invalid(DetectionContext &Context, bool Assert, Args &&...Arguments) const
Track diagnostics for invalid scops.
bool canUseISLTripCount(Loop *L, DetectionContext &Context)
Can ISL compute the trip count of a loop.
bool isCompatibleType(Instruction *Inst, llvm::Type *Ty, DetectionContext &Context)
Filter out types that we do not support.
static ScopDetection::LoopStats countBeneficialSubLoops(Loop *L, ScalarEvolution &SE, unsigned MinProfitableTrips)
Count the number of loops and the maximal loop depth in L.
#define assert(exp)
void findValues(const llvm::SCEV *Expr, llvm::ScalarEvolution &SE, llvm::SetVector< llvm::Value * > &Values)
Find the values referenced by SCEVUnknowns in a given SCEV expression.
void findLoops(const llvm::SCEV *Expr, llvm::SetVector< const llvm::Loop * > &Loops)
Find the loops referenced from a SCEV expression.
std::shared_ptr< RejectReason > RejectReasonPtr
StringRef PollySkipFnAttr
A function attribute which will cause Polly to skip the function.
bool PollyAllowFullFunction
llvm::SetVector< llvm::AssertingVH< llvm::LoadInst > > InvariantLoadsSetTy
Type for a set of invariant loads.
Definition ScopHelper.h:110
bool PollyTrackFailures
bool isAffineExpr(const llvm::Region *R, llvm::Loop *Scope, const llvm::SCEV *Expression, llvm::ScalarEvolution &SE, InvariantLoadsSetTy *ILS=nullptr)
@ Value
MemoryKind::Value: Models an llvm::Value.
Definition ScopInfo.h:150
@ PHI
MemoryKind::PHI: Models PHI nodes within the SCoP.
Definition ScopInfo.h:187
void emitRejectionRemarks(const BBPair &P, const RejectLog &Log, OptimizationRemarkEmitter &ORE)
Emit optimization remarks about the rejected regions to the user.
void getDebugLocation(const llvm::Region *R, unsigned &LineBegin, unsigned &LineEnd, std::string &FileName)
Get the location of a region from the debug info.
std::map< const Instruction *, MemAcc > MapInsnToMemAcc
const llvm::SCEV * tryForwardThroughPHI(const llvm::SCEV *Expr, llvm::Region &R, llvm::ScalarEvolution &SE, ScopDetection *SD)
Try to look through PHI nodes, where some incoming edges come from error blocks.
bool isDebugCall(llvm::Instruction *Inst)
Is the given instruction a call to a debug function?
BBPair getBBPairForRegion(const Region *R)
Return the region delimiters (entry & exit block) of R.
bool PollyProcessUnprofitable
bool isHoistableLoad(llvm::LoadInst *LInst, llvm::Region &R, llvm::LoopInfo &LI, llvm::ScalarEvolution &SE, const llvm::DominatorTree &DT, const InvariantLoadsSetTy &KnownInvariantLoads)
Check if LInst can be hoisted in R.
bool PollyUseRuntimeAliasChecks
bool PollyDelinearize
bool hasScalarDepsInsideRegion(const llvm::SCEV *Expr, const llvm::Region *R, llvm::Loop *Scope, bool AllowLoops, const InvariantLoadsSetTy &ILS)
Returns true when the SCEV contains references to instructions within the region.
bool PollyAllowUnsignedOperations
llvm::Value * getUniqueNonErrorValue(llvm::PHINode *PHI, llvm::Region *R, ScopDetection *SD)
Return a unique non-error block incoming value for PHI if available.
bool PollyInvariantLoadHoisting
bool isIgnoredIntrinsic(const llvm::Value *V)
Return true iff V is an intrinsic that we ignore during code generation.
std::pair< llvm::BasicBlock *, llvm::BasicBlock * > BBPair
Type to hold region delimiters (entry & exit block).
Definition Utils.h:31
SmallVector< const SCEV *, 4 > DelinearizedSubscripts
PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM)
ScopDetection Result
Result run(Function &F, FunctionAnalysisManager &FAM)
static AnalysisKey Key
Context variables for SCoP detection.
BaseToAFs Accesses
Map a base pointer to all access functions accessing it.
InvariantLoadsSetTy RequiredILS
Loads that need to be invariant during execution.
bool hasLoads
The region has at least one load instruction.
bool IsInvalid
If this flag is set, the SCoP must eventually be rejected, even with KeepGoing.
bool HasUnknownAccess
Flag to indicate the region has at least one unknown access.
BoxedLoopsSetTy BoxedLoopsSet
The set of loops contained in non-affine regions.
MapInsnToMemAcc InsnToMemAcc
Map to memory access description for the corresponding LLVM instructions.
RejectLog Log
Container to remember rejection reasons for this region.
RegionSet NonAffineSubRegionSet
The set of non-affine subregions in the region we analyze.
llvm::SetVector< std::pair< const SCEVUnknown *, Loop * > > NonAffineAccesses
The set of base pointers with non-affine accesses.
bool hasStores
The region has at least one store instruction.
Helper data structure to collect statistics about loop counts.
static TupleKindPtr Res
static TupleKindPtr Str
static TupleKindPtr Ctx