// Copyright 2019 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include "iree/compiler/Dialect/HAL/Transforms/Passes.h"

#include <memory>

#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "iree/compiler/Dialect/HAL/Target/Devices/LocalDevice.h"
#include "iree/compiler/Dialect/Util/Transforms/Passes.h"
#include "iree/compiler/Utils/OptionUtils.h"
#include "iree/compiler/Utils/PassUtils.h"
#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
#include "mlir/Dialect/Affine/Passes.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Pass/PassRegistry.h"
#include "mlir/Transforms/Passes.h"

namespace mlir::iree_compiler::IREE::HAL {

namespace {

struct TransformOptions : public PassPipelineOptions<TransformOptions> {
  Option<bool> serializeExecutables{
      *this,
      "serialize-executables",
      llvm::cl::desc("Whether to serialize hal.executable.variant ops to "
                     "hal.executable.binary ops."),
      llvm::cl::init(true),
  };
  Option<bool> linkExecutables{
      *this,
      "link-executables",
      llvm::cl::desc("Whether to link hal.executable ops together."),
      llvm::cl::init(true),
  };
};

static llvm::cl::opt<bool> clMemoization{
    "iree-hal-memoization",
    llvm::cl::desc(
        "Whether to memoize device resources such as command buffers."),
    llvm::cl::init(true),
};

static llvm::cl::opt<unsigned> clBenchmarkDispatchRepeatCount{
    "iree-hal-benchmark-dispatch-repeat-count",
    llvm::cl::desc(
        "The number of times to repeat each hal.command_buffer.dispatch op. "
        "This simply duplicates the dispatch op and inserts barriers. It's "
        "meant for command buffers having linear dispatch structures."),
    llvm::cl::init(1),
};

static llvm::cl::opt<llvm::cl::PowerOf2ByteSize> clInstrumentDispatchBufferSize{
    "iree-hal-instrument-dispatches",
    llvm::cl::desc("Enables dispatch instrumentation with a power-of-two byte "
                   "size used for storage (16mib, 64mib, 2gib, etc)."),
    llvm::cl::init(llvm::cl::PowerOf2ByteSize(0)),
};

static llvm::cl::list<std::string> clSubstituteExecutableSource{
    "iree-hal-substitute-executable-source",
    llvm::cl::desc(
        "A `executable_name=object_file.xxx` pair specifying a "
        "hal.executable symbol name that will be substituted with the source "
        "object file at the given path. Source object paths are relative to "
        "those specified on `--iree-hal-executable-object-search-path=`. If a "
        "`.mlir` or `.mlirbc` file is specified the entire executable will be "
        "replaced with an equivalently named hal.executable in the referenced "
        "file and otherwise the executable will be externalized and link the "
        "referenced file (`.ptx`/`.spv`/etc)."),
};

static llvm::cl::opt<std::string> clSubstituteExecutableSourcesFrom{
    "iree-hal-substitute-executable-sources-from",
    llvm::cl::desc(
        "Substitutes any hal.executable with a file in the given path with "
        "the same name ala `--iree-hal-substitute-executable-source=`."),
    llvm::cl::init(""),
};

static llvm::cl::list<std::string> clSubstituteExecutableConfiguration{
    "iree-hal-substitute-executable-configuration",
    llvm::cl::desc(
        "A `executable_name=object_file.xxx` pair specifying a hal.executable "
        "symbol name that will be substituted with the configured executable "
        "file at the given path. Configured execuable paths are relative to "
        "those specified on `--iree-hal-executable-object-search-path=`. If a "
        "`.mlir` or `.mlirbc` file is specified the entire executable will be "
        "replaced with an equivalently named hal.executable in the referenced "
        "file and otherwise the executable will be externalized and link the "
        "referenced file (`.ptx`/`.spv`/etc)."),
};

static llvm::cl::opt<std::string> clSubstituteExecutableConfigurationsFrom{
    "iree-hal-substitute-executable-configurations-from",
    llvm::cl::desc(
        "Substitutes any hal.executable with a file in the given path with "
        "the same name ala `--iree-hal-substitute-executable-configuration=`."),
    llvm::cl::init(""),
};

static llvm::cl::list<std::string> clSubstituteExecutableObject{
    "iree-hal-substitute-executable-object",
    llvm::cl::desc(
        "A `executable_name=object_file.xxx` pair specifying a "
        "hal.executable symbol name that will be substituted with the object "
        "file at the given path. Object paths are relative to those "
        "specified on `--iree-hal-executable-object-search-path=`. If a "
        "`.mlir` or `.mlirbc` file is specified the entire executable will be "
        "replaced with an equivalently named hal.executable in the referenced "
        "file and otherwise the executable will be externalized and link the "
        "referenced file (`.ptx`/`.spv`/etc)."),
};

static llvm::cl::opt<std::string> clSubstituteExecutableObjectsFrom{
    "iree-hal-substitute-executable-objects-from",
    llvm::cl::desc(
        "Substitutes any hal.executable with a file in the given path with "
        "the same name ala `--iree-hal-substitute-executable-object=`."),
    llvm::cl::init(""),
};

static llvm::cl::list<std::string> clPreprocessExecutablesWith{
    "iree-hal-preprocess-executables-with",
    llvm::cl::desc(
        "Passes each hal.executable to the given command. Multiple "
        "commands may be specified and they will be "
        "executed in order. A command may either be a pass pipeline available "
        "within the IREE compiler specified as `builtin.module(...)` or a "
        "shell tool that consumes a hal.executable MLIR file on stdin and "
        "produces a modified hal.executable on stdout. Non-zero exit codes "
        "will fail compilation."),
};

static llvm::cl::opt<bool> clLinkExecutables{
    "iree-hal-link-executables",
    llvm::cl::desc(
        "Controls linking of executables. The default is to always link, "
        "however disabling linking allows inspecting serialization "
        "of each executable in isolation and will dump a single binary per "
        "executable when used in conjunction with "
        "`--iree-hal-dump-executable-binaries-to`."),
    llvm::cl::init(true),
};

} // namespace

using FunctionLikeNest =
    MultiOpNest<func::FuncOp, IREE::Util::InitializerOp, IREE::Util::FuncOp>;

//===----------------------------------------------------------------------===//
// Utilities
//===----------------------------------------------------------------------===//

static void addCleanupPatterns(OpPassManager &passManager) {

  FunctionLikeNest(passManager)
      // Standard MLIR cleanup.
      .addPass(mlir::createCanonicalizerPass)
      .addPass(mlir::createCSEPass)

      // Simplify util.global accesses; this can help with data flow tracking as
      // redundant store-loads are removed.
      .addPass(IREE::Util::createSimplifyGlobalAccessesPass)

      // Aggressive cleanup.
      .addPass(IREE::Util::createApplyPatternsPass);

  // Cleanup and canonicalization of util.global (and other util ops).
  passManager.addPass(IREE::Util::createFoldGlobalsPass());
  passManager.addPass(IREE::Util::createFuseGlobalsPass());
}

static void addExecutableSubstitutionPasses(OpPassManager &passManager,
                                            ArrayRef<std::string> substitutions,
                                            StringRef fromPath) {
  if (!fromPath.empty()) {
    SubstituteExecutablesPassOptions substituteOptions;
    substituteOptions.searchPath = fromPath;
    passManager.addPass(
        IREE::HAL::createSubstituteExecutablesPass(substituteOptions));
  }
  if (!substitutions.empty()) {
    SubstituteExecutablesPassOptions substituteOptions;
    substituteOptions.substitutions.assign(substitutions.begin(),
                                           substitutions.end());
    passManager.addPass(
        IREE::HAL::createSubstituteExecutablesPass(substituteOptions));
  }
}

//===----------------------------------------------------------------------===//
// --iree-hal-device-assignment-pipeline
//===----------------------------------------------------------------------===//

void buildHALDeviceAssignmentPassPipeline(
    OpPassManager &passManager, const TargetRegistry &targetRegistry,
    const AssignmentOptions &assignmentOptions) {
  // The HAL must know its targets early on in the process. This pass discovers/
  // derives/specifies the target devices and annotates the module with that
  // information. This allows subsequent passes to lookup which devices they are
  // targeting.
  if (!assignmentOptions.legacyTargetBackends.empty()) {
    // Today we just assign devices from parameters but we should instead be
    // performing analysis at the flow level and then doing magic device
    // database lookups here.
    AssignLegacyTargetDevicesPassOptions options;
    options.targetRegistry = &targetRegistry;
    options.targetBackends.assign(
        assignmentOptions.legacyTargetBackends.begin(),
        assignmentOptions.legacyTargetBackends.end());
    passManager.addPass(
        IREE::HAL::createAssignLegacyTargetDevicesPass(options));
  }
  if (!assignmentOptions.targetDevices.empty()) {
    AssignTargetDevicesPassOptions options;
    options.targetDevices.assign(assignmentOptions.targetDevices.begin(),
                                 assignmentOptions.targetDevices.end());
    passManager.addPass(IREE::HAL::createAssignTargetDevicesPass(options));
  }

  // Create globals for each device (if needed).
  passManager.addPass(IREE::HAL::createMaterializeTargetDevicesPass(
      {assignmentOptions.defaultDevice}));

  // Resolve #hal.device.promise and #hal.device.alias attributes.
  passManager.addPass(IREE::HAL::createResolveDevicePromisesPass());
  passManager.addPass(
      IREE::HAL::createResolveDeviceAliasesPass({&targetRegistry}));

  // Verify devices are valid.
  passManager.addPass(IREE::HAL::createVerifyDevicesPass({&targetRegistry}));
}

//===----------------------------------------------------------------------===//
// --iree-hal-configuration-pipeline
//===----------------------------------------------------------------------===//

void buildHALConfigurationPassPipeline(OpPassManager &passManager,
                                       const TargetRegistry &targetRegistry,
                                       const TargetOptions &targetOptions,
                                       PipelineHooks hooks) {
  //----------------------------------------------------------------------------
  // Input cleanup and simplification
  //----------------------------------------------------------------------------

  // Perform cleanup upon entry so that our IR is in a good state for assignment
  // and initial interface analysis (we rely on CSE and such having been run).
  addCleanupPatterns(passManager);

  // Verify devices are valid.
  passManager.addPass(IREE::HAL::createVerifyDevicesPass({&targetRegistry}));

  //----------------------------------------------------------------------------
  // Device-specific interface materialization
  //----------------------------------------------------------------------------

  // Add dispatch instrumentation prior to materializing interfaces so we can
  // more easily mutate the stream dispatch ops and exports.
  if (auto bufferSize = clInstrumentDispatchBufferSize.getValue()) {
    passManager.addPass(IREE::HAL::createMaterializeDispatchInstrumentationPass(
        {bufferSize.value}));
  }

  // Each executable needs a hal.interface to specify how the host and
  // device communicate across the ABI boundary.
  passManager.addPass(IREE::HAL::createMaterializeInterfacesPass());

  // Prune unused executables and their contents.
  passManager.addPass(IREE::HAL::createPruneExecutablesPass());

  // Dump a source listing of each hal.executable and update the source
  // locations in the IR. This will allow us to easily inspect each executable
  // and give downstream tools that can display source information something
  // more useful and slim than the entire original source model.
  if (!targetOptions.executableSourcesPath.empty()) {
    passManager.addPass(IREE::HAL::createDumpExecutableSourcesPass(
        {targetOptions.executableSourcesPath}));
  }

  // Substitute hal.executables we've generated from earlier phases of
  // compilation with those specified on the command line. This developer
  // feature allows for splicing in hand-authored or hand-modified executables
  // in various forms without modifying the end-to-end compiler. Note that we do
  // this prior to dumping benchmarks in order to allow generating new
  // benchmarks using the substituted executables.
  addExecutableSubstitutionPasses(passManager, clSubstituteExecutableSource,
                                  clSubstituteExecutableSourcesFrom);

  // If debug information is requested capture the MLIR source text of each
  // executable variant and associate it with the entry points. This allows us
  // to preserve this information after translation and the original input IR
  // has been erased.
  if (targetOptions.debugLevel >= 3) {
    passManager.addPass(
        IREE::HAL::createCaptureExecutableSourcesPass({"0.source"}));
  }
}

//===----------------------------------------------------------------------===//
// --iree-hal-transformation-pipeline
//===----------------------------------------------------------------------===//

void buildHALTransformPassPipeline(OpPassManager &passManager,
                                   const TargetRegistry &targetRegistry,
                                   const TargetOptions &targetOptions,
                                   const TransformOptions &transformOptions,
                                   PipelineHooks hooks,
                                   PipelinePhase compileFrom,
                                   PipelinePhase compileTo) {
  //----------------------------------------------------------------------------
  // Device assignment and interface materialization
  //----------------------------------------------------------------------------

  if (hooks.beforePhase) {
    hooks.beforePhase(PipelinePhase::ExecutableSources, passManager);
  }

  if (compileFrom < PipelinePhase::ExecutableSources) {
    AssignmentOptions assignmentOptions;
    assignmentOptions.legacyTargetBackends = targetOptions.legacyTargetBackends;
    assignmentOptions.targetDevices = targetOptions.targetDevices;
    assignmentOptions.defaultDevice = targetOptions.defaultDevice;
    buildHALDeviceAssignmentPassPipeline(passManager, targetRegistry,
                                         assignmentOptions);
    buildHALConfigurationPassPipeline(passManager, targetRegistry,
                                      targetOptions, hooks);

    // Preprocess executables using an external tool. The tool may mutate one or
    // more variants and even insert or remove variants.
    for (auto command : clPreprocessExecutablesWith) {
      passManager.addNestedPass<IREE::HAL::ExecutableOp>(
          IREE::HAL::createPreprocessExecutablesPass(command));
    }
  }

  if (hooks.afterPhase) {
    hooks.afterPhase(PipelinePhase::ExecutableSources, passManager);
  }
  if (compileTo == PipelinePhase::ExecutableSources) {
    return;
  }

  //----------------------------------------------------------------------------
  // Executable translation
  //----------------------------------------------------------------------------

  if (hooks.beforePhase) {
    hooks.beforePhase(PipelinePhase::ExecutableConfigurations, passManager);
  }

  if (compileFrom < PipelinePhase::ExecutableConfigurations) {
    // Select a translation strategy for each hal.executable.variant and
    // generate the IR to condition on support for the variant. In the future,
    // this or neighboring passes can expand/contract variants based on the
    // selected translation strategies and the features each translation
    // strategy are known to require or not require.
    passManager.addNestedPass<IREE::HAL::ExecutableOp>(
        IREE::HAL::createConfigureExecutablesPass({targetRegistry}));

    // Dump a second listing of each hal.executable after preprocessing and
    // configuration of executables, as well as update locations in the IR.
    if (!targetOptions.executableConfigurationsPath.empty()) {
      passManager.addPass(IREE::HAL::createDumpExecutableSourcesPass(
          {targetOptions.executableConfigurationsPath, "configured"}));
    }

    // If debug information is requested capture the MLIR source text of each
    // configured executable variant and associate it with the entry points.
    if (targetOptions.debugLevel >= 3) {
      passManager.addPass(
          IREE::HAL::createCaptureExecutableSourcesPass({"1.configured"}));
    }

    // Substitute hal.executables we've configured with those specified on the
    // command line. This developer feature allows for hand editing the
    // configured executable with different lowering parameters.
    addExecutableSubstitutionPasses(passManager,
                                    clSubstituteExecutableConfiguration,
                                    clSubstituteExecutableConfigurationsFrom);

    // Dump standalone hal.executable benchmark modules.
    // Today this only works for executables that have static dispatch
    // parameters and is only useful for basic microbenchmarking. We do this
    // after configuration to make it easy to tweak configurations directly
    // from the benchmark.
    if (!targetOptions.executableBenchmarksPath.empty()) {
      passManager.addPass(IREE::HAL::createDumpExecutableBenchmarksPass(
          {targetOptions.executableBenchmarksPath}));
    }
  }

  if (hooks.afterPhase) {
    hooks.afterPhase(PipelinePhase::ExecutableConfigurations, passManager);
  }
  if (compileTo == PipelinePhase::ExecutableConfigurations) {
    return;
  }

  // TODO(benvanik): move translation after conversion; today translation
  // inserts the workgroup count logic we need to convert but we could instead
  // insert placeholder ops that are expanded after translation.
  //
  // Translate each executable variant to its target IR form.
  // It's extremely important this runs parallelized as it's where a large
  // majority of our compilation time lives (we invoke LLVM and lld and such).
  //
  // After this point the executables are opaque blobs and we cannot change
  // their interfaces.

  if (hooks.beforePhase) {
    hooks.beforePhase(PipelinePhase::ExecutableTargets, passManager);
  }

  if (compileFrom < PipelinePhase::ExecutableTargets) {
    passManager.addNestedPass<IREE::HAL::ExecutableOp>(
        IREE::HAL::createTranslateAllExecutablesPass({targetRegistry}));
  }

  // If debug information is requested capture the translated MLIR source text
  // of each executable variant and associate it with the entry points. This
  // allows us to compare the input IR with the translated IR before
  // serialization (LLVM dialect, SPIR-V dialect, etc).
  if (targetOptions.debugLevel >= 3) {
    passManager.addPass(
        IREE::HAL::createCaptureExecutableSourcesPass({"2.translated"}));
  }

  if (hooks.afterPhase) {
    hooks.afterPhase(PipelinePhase::ExecutableTargets, passManager);
  }
  if (compileTo == PipelinePhase::ExecutableTargets) {
    return;
  }

  // Substitute hal.executables we've translated with those specified on the
  // command line. This developer feature allows for splicing in hand-authored
  // or hand-modified executables in various forms without modifying the
  // end-to-end compiler. We support substituting prior to translation as well
  // but sometimes translation is required to produce the host code required
  // for specialization and workgroup counts and we need to perform the
  // substitution later.
  addExecutableSubstitutionPasses(passManager, clSubstituteExecutableObject,
                                  clSubstituteExecutableObjectsFrom);

  //----------------------------------------------------------------------------
  // Host program conversion
  //----------------------------------------------------------------------------

  // Convert supported input dialects (std, stream, etc) into the HAL dialect.
  passManager.addPass(IREE::HAL::createConvertToHALPass());

  // If memoization is disabled then inline any regions that were created during
  // conversion.
  if (!clMemoization) {
    FunctionLikeNest(passManager)
        .addPass(IREE::HAL::createInlineMemoizeRegionsPass);
  } else {
    passManager.addPass(IREE::HAL::createOutlineMemoizeRegionsPass());
  }

  // If any devices require the legacy synchronous execution behavior then
  // make all async operations blocking.
  passManager.addPass(IREE::HAL::createFixupLegacySyncPass());

  // Prune unused executables and their contents.
  passManager.addPass(IREE::HAL::createPruneExecutablesPass());

  addCleanupPatterns(passManager);

  //----------------------------------------------------------------------------
  // Executable packing and runtime loading
  //----------------------------------------------------------------------------

  // TODO(benvanik): move translation down to here.

  // After all executables are translated and before resolving export
  // ordinals we allow the backends to link executables together. For
  // example, the LLVM AOT backend may combine all executable targets for the
  // same architecture into a single executable and link it as a shared
  // library.
  if (transformOptions.linkExecutables && clLinkExecutables) {
    passManager.addPass(
        IREE::HAL::createLinkAllExecutablesPass({targetRegistry}));
  }

  // If any executable variants have external objects referenced within them
  // we hoist them up to the top-level variant. This is done after linking so
  // that we have the greatest chance of combining executables without different
  // object attrs preventing the merging.
  passManager.nest<IREE::HAL::ExecutableOp>()
      .addNestedPass<IREE::HAL::ExecutableVariantOp>(
          IREE::HAL::createHoistExecutableObjectsPass());

  // Resolve export ordinals from nested symbol references prior to
  // serialization. As this pass creates lookup ops it should run before
  // MaterializeResourceCachesPass.
  passManager.addPass(IREE::HAL::createResolveExportOrdinalsPass());

  // Gather cacheable resources such as executables and descriptor sets and
  // cache them at initialization-time.
  passManager.addPass(IREE::HAL::createMaterializeResourceCachesPass());

  //----------------------------------------------------------------------------
  // Device management and specialization
  //----------------------------------------------------------------------------

  // Memoize device queries such that we don't need to repeatedly ask the same
  // information at runtime.
  passManager.addPass(IREE::HAL::createMemoizeDeviceQueriesPass());

  // Big cleanup after all our conversion and materialization.
  addCleanupPatterns(passManager);

  // Benchmarking only: repeat dispatch ops a certain number of times.
  // This is guaranteed to invalidate program output and may introduce crashes
  // if there are in-place dispatches that expect specific input data.
  if (clBenchmarkDispatchRepeatCount != 1) {
    FunctionLikeNest(passManager).addPass([&]() {
      return IREE::HAL::createRepeatDispatchesPass(
          {clBenchmarkDispatchRepeatCount});
    });
  }

  // Elide redundant command buffer state ops created during conversion.
  FunctionLikeNest(passManager)
      .addPass(IREE::HAL::createElideRedundantCommandsPass);

  // Initialize device globals now that we've done the analysis that is easier
  // with them in their original target specification.
  passManager.addPass(IREE::HAL::createInitializeDevicesPass({targetRegistry}));

  // TODO: Maybe this should be a part of Affine lowering pass.
  // Remove if it is added there.
  // https://github.com/llvm/llvm-project/issues/78458
  passManager.addPass(affine::createAffineExpandIndexOpsPass());
  // Fixup workgroup count calculations that may have used the affine dialect.
  // Kind of random here but can happen if the benchmarking code does things.
  passManager.addPass(mlir::createLowerAffinePass());

  // TODO(benvanik): remove the need for this; some cleanup passes such as
  // SimplifyGlobalAccesses are currently broken with scf present.
  FunctionLikeNest(passManager).addPass(mlir::createConvertSCFToCFPass);

  //----------------------------------------------------------------------------
  // Executable serialization
  //----------------------------------------------------------------------------

  // Happens at the very end as IR is much more debuggable with the executable
  // contents not turned into a big base64 string.
  if (transformOptions.serializeExecutables) {
    passManager.addNestedPass<IREE::HAL::ExecutableOp>(
        IREE::HAL::createSerializeAllExecutablesPass(
            {&targetRegistry, targetOptions.debugLevel,
             targetOptions.executableIntermediatesPath,
             targetOptions.executableBinariesPath}));

    // NOTE: symbol DCE will destroy executable target contents, so only run
    // it if we serialized things.
    passManager.addPass(IREE::HAL::createPruneExecutablesPass());
    passManager.addPass(mlir::createSymbolDCEPass());
  }

  //----------------------------------------------------------------------------
  // Whole-program optimization
  //----------------------------------------------------------------------------

  {
    // We run these under a fixed-point iteration such that we can perform
    // inter-procedural, intra-procedural, and canonicalization as separably
    // verifiable/reusable passes. IPO will fold duplicate arguments/results
    // and inline constants to allow the local optimizations to work more
    // effectively.
    OpPassManager ipoPipeline(mlir::ModuleOp::getOperationName());

    // IPO and other cleanups.
    addCleanupPatterns(ipoPipeline);

    // Large IPO pass. Note that this can introduce a significant amount of
    // duplication/inlined constants and we'll want to ensure we're running
    // cleanup again after (this entire set of patterns is run in a
    // fixed-point iteration to do that).
    ipoPipeline.addPass(IREE::Util::createIPOPass());

    // Run fixed-point iteration on the IPO pipeline.
    passManager.addPass(
        IREE::Util::createFixedPointIteratorPass(std::move(ipoPipeline)));
  }
}

void buildHALTransformPassPipeline(OpPassManager &passManager,
                                   const TargetRegistry &targetRegistry,
                                   const TargetOptions &targetOptions,
                                   PipelineHooks hooks,
                                   PipelinePhase compileFrom,
                                   PipelinePhase compileTo) {
  TransformOptions transformOptions;
  buildHALTransformPassPipeline(passManager, targetRegistry, targetOptions,
                                transformOptions, hooks, compileFrom,
                                compileTo);
}

//===----------------------------------------------------------------------===//
// Registration
//===----------------------------------------------------------------------===//

namespace {
#define GEN_PASS_REGISTRATION
#include "iree/compiler/Dialect/HAL/Transforms/Passes.h.inc" // IWYU pragma: export
} // namespace

void registerHALPasses() {
  // Force the flags to be bound.
  // TODO(benvanik): remove the global flags and only rely on pipeline flags.
  (void)IREE::HAL::TargetOptions::FromFlags::get();
  // TODO(multi-device): move the local device registration somewhere more
  // centralized. For now we piggy-back on the pass registration as that's where
  // the local device is used.
  (void)IREE::HAL::LocalDevice::Options::FromFlags::get();
  IREE::HAL::TargetDeviceList deviceList;
  deviceList.add("local", [=]() {
    return std::make_shared<LocalDevice>(
        IREE::HAL::LocalDevice::Options::FromFlags::get());
  });
  IREE::HAL::TargetRegistry::getMutableTargetRegistry().mergeFrom(deviceList);

  // Generated.
  registerPasses();

  // Pipelines.
  PassPipelineRegistration<AssignmentOptions>(
      "iree-hal-device-assignment-pipeline",
      "Runs HAL target device assignment pipeline.",
      [](OpPassManager &passManager,
         const AssignmentOptions &assignmentOptions) {
        buildHALDeviceAssignmentPassPipeline(
            passManager, TargetRegistry::getGlobal(), assignmentOptions);
      });
  PassPipelineRegistration<>("iree-hal-configuration-pipeline",
                             "Runs HAL target configuration pipeline.",
                             [](OpPassManager &passManager) {
                               buildHALConfigurationPassPipeline(
                                   passManager, TargetRegistry::getGlobal(),
                                   TargetOptions::FromFlags::get());
                             });
  PassPipelineRegistration<TransformOptions>(
      "iree-hal-transformation-pipeline",
      "Runs the full IREE HAL conversion/lowering pipeline.",
      [](OpPassManager &passManager, const TransformOptions &transformOptions) {
        buildHALTransformPassPipeline(passManager, TargetRegistry::getGlobal(),
                                      TargetOptions::FromFlags::get(),
                                      transformOptions, PipelineHooks{},
                                      PipelinePhase::Start, PipelinePhase::End);
      });
}

} // namespace mlir::iree_compiler::IREE::HAL
