class oci::compute(
  $region_name              = 'RegionOne',
  $openstack_release        = undef,
  $cluster_name             = undef,
  $machine_hostname         = undef,
  $machine_ip               = undef,
  $bridge_mapping_list      = undef,
  $external_network_list    = undef,
  $first_master             = undef,
  $first_master_ip          = undef,
  $cluster_domain           = undef,
  $vmnet_ip                 = undef,

  $haproxy_use_sni = false,

  $glance_sni_name     = "image",
  $barbican_sni_name   = "keymanager",
  $neutron_sni_name    = "network",
  $keystone_sni_name   = "identity",

  $extswift_use_external            = false,
  $extswift_auth_url                = undef,
  $extswift_region                  = 'RegionOne',
  $extswift_proxy_url               = undef,
  $extswift_project_name            = undef,
  $extswift_project_domain_name     = 'Default',
  $extswift_user_name               = undef,
  $extswift_user_domain_name        = 'Default',
  $extswift_password                = undef,
  $cinder_backup_use_external_swift = no,

  $messaging_nodes_for_notifs = false,
  $all_rabbits                = [],
  $all_rabbits_ips            = [],


  $neutron_global_physnet_mtu = 1500,
  $neutron_path_mtu         = 1500,

  $all_masters              = undef,
  $all_masters_ip           = undef,
  $vip_hostname             = undef,
  $vip_ipaddr               = undef,
  $vip_netmask              = undef,
  $self_signed_api_cert     = true,
  $sql_vip_ip               = undef,
  $network_ipaddr           = undef,
  $network_cidr             = undef,
  $use_ssl                  = true,

  $pass_haproxy_stats       = undef,

  $pass_nova_messaging      = undef,
  $pass_nova_authtoken      = undef,
  $pass_nova_ssh_pub        = undef,
  $pass_nova_ssh_priv       = undef,
  $pass_glance_authtoken    = undef,
  $pass_neutron_authtoken   = undef,
  $pass_metadata_proxy_shared_secret = undef,
  $pass_neutron_messaging   = undef,
  $pass_neutron_vrrpauth    = undef,
  $pass_placement_authtoken = undef,
  $pass_keystone_adminuser  = undef,

  $pass_manila_db           = undef,
  $pass_manila_messaging    = undef,
  $pass_manila_authtoken    = undef,

  $has_subrole_ceilometer   = false,
  $pass_ceilometer_messaging = undef,
  $pass_ceilometer_authtoken = undef,
  $pass_ceilometer_telemetry = undef,

  $pass_cinder_db           = undef,
  $pass_cinder_authtoken    = undef,
  $pass_cinder_messaging    = undef,

  $cluster_has_osds         = undef,
  $use_ceph_if_available    = false,
  $ceph_fsid                = undef,
  $ceph_libvirtuuid         = undef,
  $ceph_bootstrap_osd_key   = undef,
  $ceph_admin_key           = undef,
  $ceph_openstack_key       = undef,
  $ceph_mon_key             = undef,
  $ceph_mon_initial_members = undef,
  $ceph_mon_host            = undef,

  $disable_notifications    = false,
  $enable_monitoring_graphs = false,
  $monitoring_graphite_host = undef,
  $monitoring_graphite_port = '2003',

  $use_gpu                  = false,
  $gpu_name                 = undef,
  $gpu_vendor_id            = undef,
  $gpu_product_id           = undef,
  $gpu_device_type          = undef,
  $vfio_ids                 = undef,

  $has_subrole_dhcp_agent   = true,

  $nested_virt              = false,

  $cpu_mode                 = 'host-passthrough',
  $cpu_model                = undef,
  $cpu_model_extra_flags    = undef,
  $hw_machine_type          = undef,

  $nova_resize_confirm_window = 0,
  $nova_enable_force_complete_and_post_copy = false,
  $nova_live_migration_downtime_steps       = 3,
  $nova_live_migration_downtime_delay       = 10,
  $nova_live_migration_completion_timeout   = 30,
  $nova_max_concurrent_live_migrations      = 3,

  $nova_run_numad = "cluster_value",

  $nova_update_resources_interval = 300,

  $nova_skip_cpu_compare_on_dest            = false,
  $nova_skip_cpu_compare_at_startup         = false,
  $nova_skip_hypervisor_version_check_on_lm = false,

  $cpu_allocation_ratio     = undef,
  $ram_allocation_ratio     = undef,
  $disk_allocation_ratio    = undef,

  $bgp_to_the_host             = false,
  $number_of_cpu_sockets       = 1,
  $dhcp_domain                 = '',
  $backup_backend              = 'ceph',
  $has_subrole_designate       = false,
  $neutron_dns_domain          = 'example.com.',
  $admin_email_address         = 'admin@example.com',
  $neutron_dnsmasq_dns_servers = '',

  $use_dynamic_routing      = false,

  $neutron_install_dragent  = false,

  $neutron_use_dvr          = true,

  $nova_reserved_host_memory_mb = 16384,
  $nova_reserved_host_disk_mb   = 10240,

  $authorized_keys_hash     = [],

  # This one is unused, because we use the agregate of
  # network + bridge instead. However, it's transmitted
  # by the ENC because it is a machine variable, so this
  # variable must be present.
  $neutron_external_network_name = undef,

  $compute_is_cephosd       = false,

  $ceph_osd_initial_setup   = false,
  $block_devices            = undef,

  $has_subrole_manila       = false,
  $manila_share_install     = false,
  $install_designate        = false,
  $install_manila           = false,
  $install_telemetry        = true,

  $manila_network_cidr          = '10.240.0.0/12',
  $manila_network_division_mask = 28,

  $kernel_from_backports    = false,
  $brex_macaddr             = undef,
  $nested_openstack         = false,

  $neutron_vxlan_vni_min     = '1000',
  $neutron_vxlan_vni_max     = '9999',

  $external_keystone_activate              = false,
  $external_keystone_url                   = 'https://api.example.com/identity',
  $external_keystone_admin_password        = undef,
  $external_keystone_region_prefixed_users = false,

  $ceph_multiple_cluster_az     = false,
  $ceph_az                      = 1,
  $ceph_multiple_az_libvirtdata = undef,
  $ceph_multiple_az_cinder_conf = undef,

  # This is the CINDER az, nothing else
  $availability_zone = 'nova',

  $configure_dkms = false,

  # libvirt tweaks
  $libvirt_max_client_requests       = 50,
  $libvirt_admin_max_client_requests = 50,
  $libvirt_max_workers               = 100,
  $libvirt_ovs_timeout               = 10,


  # Backports configuration
  $ceph_configure_osbpo       = false,
  $nova_use_qemu_backports    = false,
  $nova_use_libvirt_backports = false,

  $nova_num_pcie_ports = 16,

  $nova_swtpm_enabled = false,
  $nova_swtpm_user    = 'tss',
  $nova_swtpm_group   = 'tss',

  $rabbitmq_enable_quorum_queues = false,
  $rabbitmq_message_timeout      = 120,

  $use_json_log = false,

  $nova_run_numad_on_compute_nodes = true,

  $nova_debug_log       = false,
  $cinder_debug_log     = false,
  $ceilometer_debug_log = false,
  $neutron_debug_log    = false,
  $logrotate_hourly     = true,
){

  if $logrotate_hourly {
    $logrotate_period = 'hour'
  }else{
    $logrotate_period = 'day'
  }

  $logging_context_format_string = '%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s'
  $logging_default_format_string = '%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s'

  $rabbitmq_transient_queue_expire = $rabbitmq_message_timeout + 60

  if $rabbitmq_enable_quorum_queues {
    $rabbit_ha_queues              = false
    $rabbit_quorum_queue           = true
    $rabbit_transient_quorum_queue = true
    $amqp_durable_queues           = true
    $use_queue_manager             = true
    $rabbit_stream_fanout          = true
    $rabbit_qos_prefetch_count     = 20
  }else{
    $rabbit_ha_queues              = true
    $rabbit_quorum_queue           = false
    $rabbit_transient_quorum_queue = false
    $amqp_durable_queues           = false
    $use_queue_manager             = false
    $rabbit_stream_fanout          = false
    $rabbit_qos_prefetch_count     = $facts['os_service_default']
  }

  # Configure ceph, qemu and libvirt backports
  class { 'oci::configure_backports':
    ceph_configure_osbpo       => $ceph_configure_osbpo,
    nova_use_qemu_backports    => $nova_use_qemu_backports,
    nova_use_libvirt_backports => $nova_use_libvirt_backports,
  }

  # After Bullseye, this param is set to zero by default, and this
  # prevent post-copy to work. See:
  # https://review.opendev.org/c/openstack/nova/+/908201/4/doc/source/admin/configuring-migrations.rst#324
  sysctl::value { 'vm.unprivileged_userfaultfd':
    value  => '1',
    target => '/etc/sysctl.d/50-unprivileged-userfaultfd.conf',
  }

  # Tweak performances for KVM host
  # class { '::oci::tuned': profile => 'virtual-host' }
  if $nova_run_numad == "cluster_value" {
    if $number_of_cpu_sockets >= 2 and $nova_run_numad_on_compute_nodes {
      $do_numa = true
    }else{
      $do_numa = false
    }
  }else{
    if $number_of_cpu_sockets >= 2 and $nova_run_numad == "yes" {
      $do_numa = true
    }else{
      $do_numa = false
    }
  }

  if $do_numa {
    $packages = ['numactl','numad']
    package { $packages:
      ensure => installed,
    }
#    -> file_line { 'daemon-args-for-numad':
#      path   => '/etc/default/numad',
#      match  => '.*DAEMON_ARGS.*=.*',
#      line   => 'DAEMON_ARGS="-K 1"',
#      notify => Service['numad'],
#    }
    -> service { 'numad':
      ensure => 'running',
    }
  }else{
    $packages = ['numactl','numad']
    package { $packages:
      ensure => purged,
    }
  }

  if $configure_dkms { class { '::oci::dkms': } }

  if $nested_openstack {
    $cpu_mode_real = 'none'
    $cpu_model_real = $facts['os_service_default']
  }else{
    $cpu_mode_real = $cpu_mode
    if $cpu_mode == 'host-passthrough'{
      $cpu_model_real = $facts['os_service_default']
    }else{
      $cpu_model_real = $cpu_model
    }
  }

  if $messaging_nodes_for_notifs {
    $notif_rabbit_servers = $all_rabbits
  }else{
    $notif_rabbit_servers = $all_masters
  }

  if $use_ssl {
    $proto = 'https'
    $messaging_default_port = '5671'
    $messaging_notify_port = '5671'
    $api_port = 443
  } else {
    $proto = 'http'
    $messaging_default_port = '5672'
    $messaging_notify_port = '5672'
    $api_port = 80
  }
  $messaging_default_proto = 'rabbit'
  $messaging_notify_proto = 'rabbit'

  $base_url = "${vip_hostname}"

  $base_url_glance   = $haproxy_use_sni ? { true => "${glance_sni_name}.${vip_hostname}", false => "${base_url}/image"}
  $base_url_barbican = $haproxy_use_sni ? { true => "${barbican_sni_name}.${vip_hostname}", false => "${base_url}/keymanager"}
  $base_url_neutron  = $haproxy_use_sni ? { true => "${neutron_sni_name}.${vip_hostname}", false => "${base_url}/network"}
  $base_url_keystone = $haproxy_use_sni ? { true => "${keystone_sni_name}.${vip_hostname}", false => "${base_url}/identity"}

  $kadm_pass          = $external_keystone_activate ? { true => "${external_keystone_admin_password}", false => "${pass_keystone_adminuser}"}
  $keystone_auth_uri  = $external_keystone_activate ? { true => "${external_keystone_url}", false => "${proto}://${base_url_keystone}"}
  $keystone_admin_uri = $external_keystone_activate ? { true => "${external_keystone_url}", false => "${proto}://${base_url_keystone}"}

  $region_suffix = $external_keystone_region_prefixed_users ? {
    true  => "-${region_name}",
    false => ''
  }

  $nova_user       = "nova${region_suffix}"
  $neutron_user    = "neutron${region_suffix}"
  $placement_user  = "placement${region_suffix}"
  $cinder_user     = "cinder${region_suffix}"
  $ceilometer_user = "ceilometer${region_suffix}"
  $manila_user     = "manila${region_suffix}"

  if $self_signed_api_cert {
    $api_endpoint_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    file { "/etc/keystone":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
    }->
    file { '/etc/keystone/puppet.conf':
      ensure  => present,
      owner   => 'root',
      content => "
[keystone_authtoken]
region_name=${region_name}
auth_url=${keystone_admin_uri}
username=admin
password=${kadm_pass}
project_name=admin
interface=public
",
      mode    => '0600',
    }
  }else{
    ensure_resource('file', '/etc/openstack', {
      'ensure' => 'directory',
      'mode'   => '0755',
      'owner'  => 'root',
      'group'  => 'root',
    })
    ensure_resource('file', '/etc/openstack/puppet', {
      'ensure' => 'directory',
      'mode'   => '0755',
      'owner'  => 'root',
      'group'  => 'root',
    })
    openstacklib::clouds { '/etc/openstack/puppet/admin-clouds.yaml':
      username     => 'admin',
      password     => $kadm_pass,
      auth_url     => $keystone_admin_uri,
      project_name => 'admin',
      system_scope => 'all',
      region_name  => $region_name,
      interface    => 'public',
    }
  }

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  if $facts['os']['lsb'] != undef{
    $mycodename = $facts['os']['lsb']['distcodename']
  }else{
    $mycodename = $facts['os']['distro']['codename']
  }
  if $mycodename != 'stretch'{
    $libvirt_security_driver_content = 'security_driver = "apparmor"'
  }else{
    $libvirt_security_driver_content = '#security_driver = "apparmor"'
  }

  if $disable_notifications {
    $notification_driver = 'noop'
  }else{
    $notification_driver = 'messagingv2'
  }
  ############################################################
  ### HAProxy (used for metadata server proxying and such) ###
  ############################################################
  if $neutron_use_dvr {
    package { 'haproxy-cmd':
      ensure => present,
    }
    class { 'haproxy':
      restart_command => 'systemctl reload haproxy',
      global_options   => {
        'log'      => '/dev/log local0',
        'maxconn'  => '256',
        'user'     => 'haproxy',
        'group'    => 'haproxy',
        'daemon'   => '',
        'nbthread' => '4',
        'stats'   => [
          'socket /var/lib/haproxy/stats',
          'socket /var/lib/haproxy/admin.sock mode 600 level admin',
          'timeout 30s'
        ],
      },
      defaults_options => {
        'mode'    => 'http',
        'option'  => [
          'httplog',
        ],
      },
      merge_options => true,
    }->
    haproxy::listen { 'hapstats':
      section_name => 'statshttp',
      bind         => { "${machine_ip}:8088" => 'user root'},
      mode         => 'http',
      options      => {
          'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ],
        },
    }->
    haproxy::listen { 'haphealthcheck':
      section_name => 'healthcheck',
      bind         => { "${machine_ip}:8081" => 'user root'},
      mode         => 'http',
      options => [
        { 'option' => 'httpchk' },
        { 'option' => 'http-keep-alive' },

      ]
    }

    $haproxy_options_for_metadata_proxy = [
      { 'use_backend' => 'metadatabe' },
    ]
    haproxy::frontend { 'openstackfe':
      mode      => 'http',
      bind      => { "127.0.0.1:8775" => [] },
      options   => $haproxy_options_for_metadata_proxy,
    }
    haproxy::backend { 'metadatabe':
      options => [
        { 'option' => 'forwardfor' },
        { 'option' => 'httpchk GET /healthcheck' },
        { 'mode' => 'http' },
        { 'balance' => 'roundrobin' },
      ],
    }
    if $openstack_release == 'rocky'{
      $metadatabm_options = 'check'
    }else{
      $metadatabm_options = 'check check-ssl ssl verify required ca-file /etc/ssl/certs/oci-pki-oci-ca-chain.pem'
    }
    haproxy::balancermember { 'metadatabm':
      listening_service => 'metadatabe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8775,
      options           => $metadatabm_options
    }
  } else {
    package { 'remove-haproxy-in-non-dvr':
      name   => 'haproxy',
      ensure => purged,
    }
  }

  ############
  ### Nova ###
  ############
  if $disable_notifications {
    $nova_notif_transport_url = undef
  } else {
    $nova_notif_transport_url = os_transport_url({
                                  'transport' => $messaging_notify_proto,
                                  'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                  'port'      => $messaging_notify_port,
                                  'username'  => 'nova',
                                  'password'  => $pass_nova_messaging,
                                })
  }

  # This is needed for Buster, so live migration can work
  package { 'apparmor':
    ensure => present,
    before => Class['::nova'],
  }

  # This *must* be called either before the main "::nova" class (latest puppet-nova versions)
  # or before "::nova::conductor" (before Wallby) as there's an include.
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga') {
    nova_config {
      'workarounds/skip_hypervisor_version_check_on_lm': value => $nova_skip_hypervisor_version_check_on_lm;
      'workarounds/skip_cpu_compare_on_dest':            value => $nova_skip_cpu_compare_on_dest;
      'workarounds/skip_cpu_compare_at_startup':         value => $nova_skip_cpu_compare_at_startup;
    }
  }elsif($openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat' or $openstack_release == 'caracal' or $openstack_release == 'dalmatian' or $openstack_release == 'epoxy'){
      class { '::nova::workarounds':
        skip_hypervisor_version_check_on_lm => $nova_skip_hypervisor_version_check_on_lm,
      }
      nova_config {
        'workarounds/skip_cpu_compare_on_dest': value => $nova_skip_cpu_compare_on_dest;
        'workarounds/skip_cpu_compare_at_startup': value => $nova_skip_cpu_compare_at_startup;
      }
  # Flamingo and up.
  }else{
    class { '::nova::workarounds':
      skip_hypervisor_version_check_on_lm => $nova_skip_hypervisor_version_check_on_lm,
      skip_cpu_compare_on_dest            => $nova_skip_cpu_compare_on_dest,
      skip_cpu_compare_at_startup         => $nova_skip_cpu_compare_at_startup,
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::nova':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'nova',
        'password'  => $pass_nova_messaging,
      }),
      notification_transport_url  => $nova_notif_transport_url,
      database_connection         => '',
      rabbit_use_ssl              => $use_ssl,
      rpc_response_timeout        => $rabbitmq_message_timeout,
      amqp_durable_queues         => false,
      amqp_auto_delete            => false,
      rabbit_ha_queues            => true,
      kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms        => 'PLAIN',
      glance_api_servers          => "${proto}://${base_url_glance}",
      notification_driver         => $notification_driver,
      notify_on_state_change      => 'vm_and_task_state',
      nova_public_key             => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
      nova_private_key            => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
      cpu_allocation_ratio        => $cpu_allocation_ratio,
      ram_allocation_ratio        => $ram_allocation_ratio,
      disk_allocation_ratio       => $disk_allocation_ratio,
      rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
      my_ip                       => $machine_ip,
    }
  }else{
    if $openstack_release == 'victoria'{
      class { '::nova':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'nova',
          'password'  => $pass_nova_messaging,
        }),
        notification_transport_url             => $nova_notif_transport_url,
        database_connection                    => '',
        rabbit_use_ssl                         => $use_ssl,
        rpc_response_timeout                   => $rabbitmq_message_timeout,
        amqp_durable_queues                    => false,
        amqp_auto_delete                       => false,
        rabbit_ha_queues                       => true,
        kombu_ssl_ca_certs                     => $oci_pki_root_ca_file,
        notification_driver                    => $notification_driver,
        notify_on_state_change                 => 'vm_and_task_state',
        nova_public_key                        => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
        nova_private_key                       => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
        cpu_allocation_ratio                   => $cpu_allocation_ratio,
        ram_allocation_ratio                   => $ram_allocation_ratio,
        disk_allocation_ratio                  => $disk_allocation_ratio,
        glance_endpoint_override               => "${proto}://${base_url_glance}",
        block_device_allocate_retries          => 600,
        block_device_allocate_retries_interval => 6,
        rabbit_transient_queues_ttl            => $rabbitmq_transient_queue_expire,
        my_ip                                  => $machine_ip,
      }
    }elsif $openstack_release == 'wallaby' or $openstack_release == 'xena'{
      class { '::nova':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'nova',
          'password'  => $pass_nova_messaging,
        }),
        notification_transport_url  => $nova_notif_transport_url,
        database_connection         => '',
        rabbit_use_ssl              => $use_ssl,
        rpc_response_timeout        => $rabbitmq_message_timeout,
        amqp_durable_queues         => false,
        amqp_auto_delete            => false,
        rabbit_ha_queues            => true,
        kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
        notification_driver         => $notification_driver,
        notify_on_state_change      => 'vm_and_task_state',
        nova_public_key             => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
        nova_private_key            => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
        cpu_allocation_ratio        => $cpu_allocation_ratio,
        ram_allocation_ratio        => $ram_allocation_ratio,
        disk_allocation_ratio       => $disk_allocation_ratio,
        glance_endpoint_override    => "${proto}://${base_url_glance}",
        rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
        my_ip                       => $machine_ip,
      }
    # Yoga and up
    }else{
# We can't call the class like this anymore, because
# the class checks for a patern.
#      class { '::nova::db':
#        database_connection    => '',
#      }
      if($openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
        class { '::nova':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'nova',
            'password'  => $pass_nova_messaging,
          }),
          notification_transport_url  => $nova_notif_transport_url,
          rabbit_use_ssl              => $use_ssl,
          rpc_response_timeout        => $rabbitmq_message_timeout,
          amqp_durable_queues         => false,
          amqp_auto_delete            => false,
          rabbit_ha_queues            => true,
          kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
          notification_driver         => $notification_driver,
          notify_on_state_change      => 'vm_and_task_state',
          nova_public_key             => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
          nova_private_key            => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
          cpu_allocation_ratio        => $cpu_allocation_ratio,
          ram_allocation_ratio        => $ram_allocation_ratio,
          disk_allocation_ratio       => $disk_allocation_ratio,
          rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
          my_ip                       => $machine_ip,
        }
      }elsif($openstack_release == 'bobcat'){
        class { '::nova':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'nova',
            'password'  => $pass_nova_messaging,
          }),
          notification_transport_url  => $nova_notif_transport_url,
          rabbit_use_ssl              => $use_ssl,
          rpc_response_timeout        => $rabbitmq_message_timeout,
          amqp_durable_queues         => $amqp_durable_queues,
          amqp_auto_delete            => false,
          rabbit_ha_queues            => $rabbit_ha_queues,
          rabbit_quorum_queue         => $rabbit_quorum_queue,
          kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
          notification_driver         => $notification_driver,
          notify_on_state_change      => 'vm_and_task_state',
          nova_public_key             => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
          nova_private_key            => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
          cpu_allocation_ratio        => $cpu_allocation_ratio,
          ram_allocation_ratio        => $ram_allocation_ratio,
          disk_allocation_ratio       => $disk_allocation_ratio,
          rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
          my_ip                       => $machine_ip,
        }
      # Caracal and up.
      }else{
        class { '::nova':
          default_transport_url         => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'nova',
            'password'  => $pass_nova_messaging,
          }),
          notification_transport_url    => $nova_notif_transport_url,
          rabbit_use_ssl                => $use_ssl,
          rpc_response_timeout          => $rabbitmq_message_timeout,
          amqp_durable_queues           => $amqp_durable_queues,
          amqp_auto_delete              => false,
          rabbit_ha_queues              => $rabbit_ha_queues,
          rabbit_quorum_queue           => $rabbit_quorum_queue,
          rabbit_transient_quorum_queue => $rabbit_transient_quorum_queue,
          rabbit_use_queue_manager      => $use_queue_manager,
          rabbit_stream_fanout          => $rabbit_stream_fanout,
          rabbit_qos_prefetch_count     => $rabbit_qos_prefetch_count,
          kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
          notification_driver           => $notification_driver,
          notify_on_state_change        => 'vm_and_task_state',
          nova_public_key               => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
          nova_private_key              => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
          cpu_allocation_ratio          => $cpu_allocation_ratio,
          ram_allocation_ratio          => $ram_allocation_ratio,
          disk_allocation_ratio         => $disk_allocation_ratio,
          rabbit_transient_queues_ttl   => $rabbitmq_transient_queue_expire,
          my_ip                         => $machine_ip,
        }
      }
    }
    nova_config {
      'glance/api_servers': value => "${proto}://${base_url_glance}";
      'glance/region_name': value => $region_name;
    }
  }

  class { '::nova::keystone::authtoken':
    username             => $nova_user,
    password             => $pass_nova_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    cafile               => $api_endpoint_ca_file,
    region_name          => $region_name,
  }

  # This is mandatory starting with bobcat, and this is supported in puppet-nova only in train and up.
  # This is related to CVE-2023-2088.
  if ($openstack_release != 'rocky' and $openstack_release != 'stein'){
    class { '::nova::keystone::service_user':
      username                => $nova_user,
      password                => $pass_nova_authtoken,
      auth_url                => $keystone_admin_uri,
      project_name            => 'services',
      region_name             => $region_name,
      send_service_user_token => true,
      cafile                  => $api_endpoint_ca_file,
    }
  }

  if $openstack_release != 'rocky' and $openstack_release != 'stein' {
    if $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga'{
      class { '::nova::cinder':
        username       => $cinder_user,
        password       => $pass_cinder_authtoken,
        region_name    => $region_name,
        os_region_name => $region_name,
        auth_url       => $keystone_admin_uri,
      }
    # Zed and up
    }else{
      class { '::nova::cinder':
        username       => $cinder_user,
        password       => $pass_cinder_authtoken,
        os_region_name => $region_name,
        auth_url       => $keystone_admin_uri,
      }
    }
  }


  # Needed, so puppet-openstack can restart virtlogd and libvirtd after configuration
  # so it listen to TCP before nova-compute starts.
  # This is called with modular_libvirt == false, because of a patch in the debian package for puppet-nova that re-enables it,
  # but does not account for the variable having side effects on libvirtd service lifecycle.
  class { '::nova::compute::libvirt::services':
    modular_libvirt => false,
  }
  Service['virtlogd'] -> Service['libvirtd']

  # Tweak libvirt defaults to allow more parallel operations
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga') {
    class { '::nova::compute::libvirt::config':
      libvirtd_config => {
        'max_client_requests'       => { value => $libvirt_max_client_requests},
        'admin_max_client_requests' => { value => $libvirt_admin_max_client_requests},
        'max_workers'               => { value => $libvirt_max_workers},
      }
    }
  }else{
    # Before Yoga, ::nova::compute::libvirt has the ovs_timeout param and conflicts.
    class { '::nova::compute::libvirt::config':
      libvirtd_config => {
        'max_client_requests'       => { value => $libvirt_max_client_requests},
        'admin_max_client_requests' => { value => $libvirt_admin_max_client_requests},
        'max_workers'               => { value => $libvirt_max_workers},
        'ovs_timeout'               => { value => $libvirt_ovs_timeout},
      }
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    # This enables live migrations over libvirt's TLS support
    class { '::nova::migration::libvirt':
      transport                      => 'tls',
      live_migration_inbound_addr    => $machine_hostname,
      live_migration_tunnelled       => false,
      live_migration_with_native_tls => true,
      ca_file                        => '/etc/pki/CA/cacert.pem',
    }
  }else{
    if $nova_enable_force_complete_and_post_copy {
      $live_migration_timeout_action     = 'force_complete'
      $live_migration_permit_post_copy   = true
      $live_migration_downtime_steps     = $nova_live_migration_downtime_steps
      $live_migration_downtime_delay     = $nova_live_migration_downtime_delay
      $live_migration_completion_timeout = $nova_live_migration_completion_timeout
    }else{
      $live_migration_timeout_action     = $facts['os_service_default']
      $live_migration_permit_post_copy   = $facts['os_service_default']
      $live_migration_downtime_steps     = $facts['os_service_default']
      $live_migration_downtime_delay     = $facts['os_service_default']
      $live_migration_completion_timeout = $facts['os_service_default']
    }
    # This enables live migrations over libvirt's TLS support
    if ($openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat') {
      class { '::nova::migration::libvirt':
        transport                         => 'tls',
        live_migration_inbound_addr       => $machine_hostname,
        live_migration_tunnelled          => false,
        live_migration_with_native_tls    => true,
        ca_file                           => '/etc/pki/CA/cacert.pem',
        modular_libvirt                   => true,
        live_migration_timeout_action     => $live_migration_timeout_action,
        live_migration_permit_post_copy   => $live_migration_permit_post_copy,
        live_migration_downtime_steps     => $live_migration_downtime_steps,
        live_migration_downtime_delay     => $live_migration_downtime_delay,
        live_migration_completion_timeout => $nova_live_migration_completion_timeout,
      }
    # Dalmatian and up need key_file and cert_file params.
    }else{
      class { '::nova::migration::libvirt':
        transport                         => 'tls',
        live_migration_inbound_addr       => $machine_hostname,
        live_migration_tunnelled          => false,
        live_migration_with_native_tls    => true,
        ca_file                           => '/etc/pki/CA/cacert.pem',
        modular_libvirt                   => true,
        key_file                          => '/etc/pki/libvirt/private/serverkey.pem',
        cert_file                         => '/etc/pki/libvirt/servercert.pem',
        live_migration_timeout_action     => $live_migration_timeout_action,
        live_migration_permit_post_copy   => $live_migration_permit_post_copy,
        live_migration_downtime_steps     => $live_migration_downtime_steps,
        live_migration_downtime_delay     => $live_migration_downtime_delay,
        live_migration_completion_timeout => $nova_live_migration_completion_timeout,
      }
    }
  }

  if $nested_openstack {
    $virt_type = 'qemu'
  }else{
    $virt_type = 'kvm'
  }

#  if $use_ceph_if_available or $compute_is_cephosd {
#    $disk_cachemodes = ['network=writeback', ]
#  }else{
#    $disk_cachemodes = $facts['os_service_default']
#  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::nova::compute::libvirt':
#      disk_cachemodes                            => $disk_cachemodes,
      libvirt_virt_type                          => $virt_type,
      libvirt_cpu_mode                           => $cpu_mode_real,
      libvirt_cpu_model                          => $cpu_model_real,
      libvirt_cpu_model_extra_flags              => $cpu_model_extra_flags,
      libvirt_hw_machine_type                    => $hw_machine_type,
      migration_support                          => true,
      # False is needed because of the above include ::nova::compute::libvirt::services
      manage_libvirt_services                    => false,
      preallocate_images                         => 'space',
      # This is one week retention.
      remove_unused_original_minimum_age_seconds => '604800',
      vncserver_listen                           => '0.0.0.0',
      libvirt_hw_disk_discard                    => 'unmap',
      num_pcie_ports                             => $nova_num_pcie_ports,
    }
  }else{
    if ($openstack_release == 'victoria' or $openstack_release == 'wallaby') {
      class { '::nova::compute::libvirt':
#        disk_cachemodes                            => $disk_cachemodes,
        virt_type                                  => $virt_type,
        cpu_mode                                   => $cpu_mode_real,
        cpu_models                                 => [ $cpu_model_real, ],
        cpu_model_extra_flags                      => $cpu_model_extra_flags,
        hw_machine_type                            => $hw_machine_type,
        migration_support                          => true,
        # False is needed because of the above include ::nova::compute::libvirt::services
        manage_libvirt_services                    => false,
        preallocate_images                         => 'space',
        # This is one week retention.
        remove_unused_original_minimum_age_seconds => '604800',
        vncserver_listen                           => '0.0.0.0',
        libvirt_hw_disk_discard                    => 'unmap',
        num_pcie_ports                             => $nova_num_pcie_ports,
      }
    # Xena and up
    }else{
      if $use_ceph_if_available or $compute_is_cephosd{
        $images_type = 'rbd'
      }else{
        $images_type = $facts['os_service_default']
      }
      if $cpu_model_real == $facts['os_service_default'] {
        $cpu_models_param = undef
      }else{
        $cpu_models_param = [ $cpu_model_real, ]
      }
      class { '::nova::compute::libvirt':
#        disk_cachemodes                            => $disk_cachemodes,
        virt_type                                  => $virt_type,
        cpu_mode                                   => $cpu_mode_real,
        cpu_models                                 => $cpu_models_param,
        cpu_model_extra_flags                      => $cpu_model_extra_flags,
        hw_machine_type                            => $hw_machine_type,
        migration_support                          => true,
        # False is needed because of the above include ::nova::compute::libvirt::services
        manage_libvirt_services                    => false,
        preallocate_images                         => 'space',
        # This is one week retention.
        vncserver_listen                           => '0.0.0.0',
        hw_disk_discard                            => 'unmap',
        images_type                                => $images_type,
        num_pcie_ports                             => $nova_num_pcie_ports,
        swtpm_enabled                              => $nova_swtpm_enabled,
        swtpm_user                                 => $nova_swtpm_user,
        swtpm_group                                => $nova_swtpm_group,
      }
      class { '::nova::compute::image_cache':
        remove_unused_original_minimum_age_seconds => '604800',
      }
    }
  }
  # Starting with Caracal, nova::migration::libvirt has 2 new params:
  # cert_file and key_file that we should use instead of setting this
  # up "manually"
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat') {
    create_resources( 'libvirtd_config', {
      'key_file'=>  { 'value' => '/etc/pki/libvirt/private/serverkey.pem', 'quote'  => true },
      'cert_file'=> { 'value' => '/etc/pki/libvirt/servercert.pem', 'quote'  => true },
    })
  }

  # The +0 is there for converting the string to an int,
  # 1536 is for the PoC, 8192 is for real deployments where we do expect
  # hosts to hold more than 16GB of RAM.
  if (($facts['memory']['system']['available_bytes'] + 0) < (16 * 1024 * 1024 * 1024)) {
    $reserved_host_memory_mb = 4096
  }else{
    $reserved_host_memory_mb = $nova_reserved_host_memory_mb
  }

  nova_config {
    'placement/cafile':                 value => $api_endpoint_ca_file;
    'placement/auth_type':              value => 'password';
    'placement/auth_url':               value => $keystone_admin_uri;
    'placement/password':               value => $pass_placement_authtoken, secret => true;
    'placement/project_domain_name':    value => 'Default';
    'placement/project_name':           value => 'services';
    'placement/user_domain_name':       value => 'Default';
    'placement/username':               value => $placement_user;
    'placement/region_name':            value => $region_name;
    'glance/cafile':                    value => $api_endpoint_ca_file;
    'keystone/cafile':                  value => $api_endpoint_ca_file;
    'neutron/cafile':                   value => $api_endpoint_ca_file;
    'cinder/cafile':                    value => $api_endpoint_ca_file;
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria') {
    nova_config {
      'DEFAULT/use_cow_images':           value => 'False';
      'DEFAULT/default_ephemeral_format': value => 'ext4';
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    nova_config {
      'DEFAULT/reserved_host_disk_mb':    value => $nova_reserved_host_disk_mb;
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    if $openstack_release != 'rocky' and $openstack_release != 'stein'{
      class { '::nova::compute::image_cache':
        manager_interval => '-1';
      }

    }
  }
  if $openstack_release != 'rocky' and $openstack_release != 'stein'{
    # Every 6 hours, clean the _base image folder of Nova, otherwise it
    # may become quickly super big.
    cron { 'oci-compute-node-image-cache-cleaner':
      command => '/usr/bin/flock -w 0 /var/lock/oci-compute-node-image-cache-cleaner.lock /usr/bin/oci-compute-node-image-cache-cleaner >/dev/null 2>&1',
      user    => 'nova',
      hour    => [0, 6, 12, 18],
      require => Anchor['nova::install::end'],
    }
  }

  # Note: this *must* be set *before* calling ::nova::compute, as ::nova::compute is
  # doing an include ::nova::pci.
  if $use_gpu {

    $aliases_array = $gpu_name.map |Integer $i, String $n| {
      {"vendor_id" => $gpu_vendor_id[$i], "product_id" => $gpu_product_id[$i], "name" => $gpu_name[$i], "device_type" => $gpu_device_type}
    }
    $passthrough_array = $gpu_name.map |Integer $i, String $n| {
      {"vendor_id" => $gpu_vendor_id[$i], "product_id" => $gpu_product_id[$i]}
    }
    #notify { "GPU aliases array: ${aliases_array}": withpath => true }
    #notify { "GPU passthrough array: ${passthrough_array}": withpath => true }

    class { '::nova::pci':
      aliases => $aliases_array,
    }
    class {'::nova::compute::pci':
      passthrough => $passthrough_array,
    }
    file { '/etc/modprobe.d/blacklist-nvidia.conf':
      ensure                  => present,
      owner                   => 'root',
      content                 => 'blacklist nouveau
blacklist nvidiafb
blacklist snd_hda_intel
',
      selinux_ignore_defaults => true,
      mode                    => '0644',
    }
    file { '/etc/modprobe.d/vfio.conf':
      ensure                  => present,
      owner                   => 'root',
      content                 => "options vfio-pci ids=${vfio_ids}",
      selinux_ignore_defaults => true,
      mode                    => '0644',
    }
  }else{
    file { '/etc/modprobe.d/blacklist-nvidia.conf':
      ensure                  => absent,
    }
    file { '/etc/modprobe.d/vfio.conf':
      ensure                  => absent,
    }
  }
  # Needed to expose physical devices to VMs
  file { '/etc/modules-load.d/vfio.conf':
    ensure                  => present,
    owner                   => 'root',
    content                 => "# Managed by puppet: do not touch

vfio-pci
",
    selinux_ignore_defaults => true,
    mode                    => '0644',
  }

  if $openstack_release == 'rocky'{
    class { '::nova::compute':
      vnc_enabled                      => true,
      vncproxy_host                    => "${vip_hostname}",
      vncproxy_protocol                => 'https',
      vncproxy_port                    => '443',
      vncproxy_path                    => '/novnc/vnc_auto.html',
      vncserver_proxyclient_address    => $machine_ip,
      instance_usage_audit             => true,
      instance_usage_audit_period      => 'hour',
      barbican_auth_endpoint           => "${keystone_auth_uri}/v3",
      barbican_endpoint                => "${proto}://${base_url_barbican}",
      keymgr_backend                   => 'barbican',
      resume_guests_state_on_host_boot => true,
      allow_resize_to_same_host        => true,
      force_config_drive               => true,
      reserved_host_memory             => $reserved_host_memory_mb,
      resize_confirm_window            => $nova_resize_confirm_window,
    }
    nova_config {
      'barbican/barbican_region_name':    value => $region_name;
    }
  } else {
    if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::nova::compute':
        vnc_enabled                      => true,
        vncproxy_host                    => "${vip_hostname}",
        vncproxy_protocol                => 'https',
        vncproxy_port                    => '443',
        vncproxy_path                    => '/novnc/vnc_auto.html',
        vncserver_proxyclient_address    => $machine_ip,
        instance_usage_audit             => true,
        instance_usage_audit_period      => 'hour',
        barbican_auth_endpoint           => "${keystone_auth_uri}/v3",
        barbican_endpoint                => "${proto}://${base_url_barbican}",
        keymgr_backend                   => 'barbican',
        resume_guests_state_on_host_boot => true,
        allow_resize_to_same_host        => true,
        force_config_drive               => true,
        reserved_host_memory             => $reserved_host_memory_mb,
        resize_confirm_window            => $nova_resize_confirm_window,
      }
      nova_config {
        'barbican/barbican_region_name':    value => $region_name;
      }
    }else{
      if $openstack_release == 'victoria' {
        class { '::nova::compute':
          vnc_enabled                      => true,
          vncproxy_host                    => "${vip_hostname}",
          vncproxy_protocol                => 'https',
          vncproxy_port                    => '443',
          vncproxy_path                    => '/novnc/vnc_auto.html',
          vncserver_proxyclient_address    => $machine_ip,
          instance_usage_audit             => true,
          instance_usage_audit_period      => 'hour',
          barbican_auth_endpoint           => "${keystone_auth_uri}/v3",
          barbican_endpoint                => "${proto}://${base_url_barbican}",
          keymgr_backend                   => 'barbican',
          resume_guests_state_on_host_boot => true,
          allow_resize_to_same_host        => true,
          force_config_drive               => true,
          reserved_host_memory             => $reserved_host_memory_mb,
          reserved_host_disk               => $nova_reserved_host_disk_mb,
          resize_confirm_window            => $nova_resize_confirm_window,
          update_resources_interval        => $nova_update_resources_interval,
          max_concurrent_live_migrations   => $nova_max_concurrent_live_migrations,
        }
        nova_config {
          'barbican/barbican_region_name':    value => $region_name;
        }
      }elsif $openstack_release == 'wallaby'{
        class { '::nova::compute':
          vnc_enabled                      => true,
          vncproxy_host                    => "${vip_hostname}",
          vncproxy_protocol                => 'https',
          vncproxy_port                    => '443',
          vncproxy_path                    => '/novnc/vnc_auto.html',
          vncserver_proxyclient_address    => $machine_ip,
          instance_usage_audit             => true,
          instance_usage_audit_period      => 'hour',
          barbican_auth_endpoint           => "${keystone_auth_uri}/v3",
          barbican_endpoint                => "${proto}://${base_url_barbican}",
          keymgr_backend                   => 'barbican',
          resume_guests_state_on_host_boot => true,
          force_config_drive               => true,
          reserved_host_memory             => $reserved_host_memory_mb,
          reserved_host_disk               => $nova_reserved_host_disk_mb,
          use_cow_images                   => 'False',
          default_ephemeral_format         => 'ext4',
          resize_confirm_window            => $nova_resize_confirm_window,
          block_device_allocate_retries    => 600,
          block_device_allocate_retries_interval => 6,
          update_resources_interval        => $nova_update_resources_interval,
          max_concurrent_live_migrations   => $nova_max_concurrent_live_migrations,
        }
        nova_config {
          'barbican/barbican_region_name':    value => $region_name;
        }
      }else{
        class { '::nova::key_manager::barbican':
          barbican_endpoint    => "${proto}://${base_url_barbican}",
          auth_endpoint        => "${keystone_auth_uri}/v3",
          barbican_region_name => "${region_name}",
        }
        class { '::nova::compute':
          vnc_enabled                      => true,
          vncproxy_host                    => "${vip_hostname}",
          vncproxy_protocol                => 'https',
          vncproxy_port                    => 443,
          vncproxy_path                    => '/novnc/vnc_auto.html',
          vncserver_proxyclient_address    => $machine_ip,
          instance_usage_audit             => true,
          instance_usage_audit_period      => 'hour',
          resume_guests_state_on_host_boot => true,
          force_config_drive               => true,
          reserved_host_memory             => $reserved_host_memory_mb,
          reserved_host_disk               => $nova_reserved_host_disk_mb,
          use_cow_images                   => 'False',
          default_ephemeral_format         => 'ext4',
          block_device_allocate_retries    => 600,
          block_device_allocate_retries_interval => 6,
          resize_confirm_window            => $nova_resize_confirm_window,
          update_resources_interval        => $nova_update_resources_interval,
          max_concurrent_live_migrations   => $nova_max_concurrent_live_migrations,
        }
      }
    }
    class { '::nova::logging':
      debug                         => $nova_debug_log,
      use_json                      => $use_json_log,
      logging_context_format_string => $logging_context_format_string,
      logging_default_format_string => $logging_default_format_string,
    }
  }

  file_line { 'parallel-shutdown-of-vms':
    path    => '/etc/default/libvirt-guests',
    match   => '.*PARALLEL_SHUTDOWN.*=.*',
    line    => 'PARALLEL_SHUTDOWN=8',
    require => Class['::nova::compute::libvirt'],
  }->
  file_line { 'shutdown-timeout-of-vms':
    path   => '/etc/default/libvirt-guests',
    match  => '.*SHUTDOWN_TIMEOUT.*=.*',
    line   => 'SHUTDOWN_TIMEOUT=120',
  }->
  file_line { 'start-delay-of-vms':
    path   => '/etc/default/libvirt-guests',
    match  => '.*START_DELAY.*=.*',
    line   => 'START_DELAY=4',
  }->
  file_line { 'use-apparmor-not-selinux':
    path   => '/etc/libvirt/qemu.conf',
    match  => '^#?security_driver.*=.*',
    line   => $libvirt_security_driver_content,
    notify => Service['libvirtd'],
  }->
  file_line { 'enable-vnc-tls':
    path   => '/etc/libvirt/qemu.conf',
    match  => '^#?vnc_tls[ \t]*=.*',
    line   => 'vnc_tls=1',
    notify => Service['libvirtd'],
  }->
  file_line { 'vnc-tls-x509-verify':
    path   => '/etc/libvirt/qemu.conf',
    match  => '^#?vnc_tls_x509_verify[ \t]*=.*',
    line   => 'vnc_tls_x509_verify=1',
    notify => Service['libvirtd'],
  }

  if 'sev' in $facts['oci_cpu_flags'] {
    $sev = ' sev=1'
  }else{
    $sev = ''
  }

  if $nested_virt{
    file { '/etc/modprobe.d/kvm.conf':
      ensure  => present,
      content => 'options kvm_intel nested=1',
    }
    file { '/etc/modprobe.d/kvm_amd.conf':
      ensure  => present,
      content => "options kvm_amd nested=1${sev}",
    }
  }else{
    file { '/etc/modprobe.d/kvm.conf':
      ensure  => present,
      content => 'options kvm_intel nested=0',
    }
    file { '/etc/modprobe.d/kvm_amd.conf':
      ensure  => present,
      content => "options kvm_amd nested=0${sev}",
    }
  }

  if $openstack_release == 'rocky'{
    class { '::nova::network::neutron':
      neutron_auth_url      => "${keystone_auth_uri}/v3",
      neutron_url           => "${proto}://${base_url_neutron}",
      neutron_username      => $neutron_user,
      neutron_password      => $pass_neutron_authtoken,
      default_floating_pool => 'public',
      dhcp_domain           => $dhcp_domain,
      neutron_region_name   => $region_name,
    }
    nova_config {
      'neutron/endpoint_override': value => "${proto}://${base_url_neutron}";
    }
  }else{
    if $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::nova::network::neutron':
        neutron_auth_url          => "${keystone_auth_uri}/v3",
        neutron_url               => "${proto}://${base_url_neutron}",
        neutron_username          => $neutron_user,
        neutron_password          => $pass_neutron_authtoken,
        default_floating_pool     => 'public',
        dhcp_domain               => $dhcp_domain,
        neutron_endpoint_override => "${proto}://${base_url_neutron}",
        neutron_region_name       => $region_name,
      }
    }else{
      class { '::nova::network::neutron':
        auth_url              => "${keystone_auth_uri}/v3",
        username              => $neutron_user,
        password              => $pass_neutron_authtoken,
        default_floating_pool => 'public',
        endpoint_override     => "${proto}://${base_url_neutron}",
        region_name           => $region_name,
      }
      class { '::nova::metadata':
        dhcp_domain                          => $dhcp_domain,
        neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
      }
    }
  }

  if $cluster_has_osds {
    class { 'ceph':
      fsid                => $ceph_fsid,
      ensure              => 'present',
      authentication_type => 'cephx',
      mon_initial_members => $ceph_mon_initial_members,
      mon_host            => $ceph_mon_host,
    }->
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mds => 'allow',
    }->
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
    }->
    ceph::key { 'client.bootstrap-osd':
      secret       => $ceph_bootstrap_osd_key,
      keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
      cap_mon      => 'allow profile bootstrap-osd',
    }->
    class { '::ceph::profile::params':
      fsid                => $ceph_fsid,
      manage_repo         => false,
      release             => 'nautilus',
      authentication_type => 'cephx',
      mon_host            => $ceph_mon_host,
      mon_initial_members => $ceph_mon_initial_members,
      client_keys         => {
        'client.admin' => {
          secret  => $ceph_admin_key,
          cap_mon => 'allow *',
          cap_osd => 'allow *',
          cap_mds => 'allow',
        },
        'client.openstack' => {
          secret  => $ceph_openstack_key,
          mode    => '0644',
          cap_mon => 'profile rbd',
          cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
        },
        'client.bootstrap-osd' => {
          secret       => $ceph_bootstrap_osd_key,
          keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
          cap_mon      => 'allow profile bootstrap-osd',
        }
      }
    }

    if $ceph_multiple_cluster_az and $ceph_multiple_az_cinder_conf {
      $ceph_multiple_az_cinder_conf.each |Hash $conf| {
        $az_id                    = $conf['az']
        $ceph_fsid                = $conf['fsid']
        $ceph_mon_host            = $conf['mon_host']
        $ceph_mon_initial_members = $conf['mon_initial_members']
        $ceph_admin_key           = $conf['admin_key']
        $ceph_openstack_key       = $conf['openstack_key']
        $ceph_bootstrap_osd_key   = $conf['boostrap_osd_key']

        $ceph_file_name_base = "/etc/ceph/ceph-az${az_id}"

        $ceph_conf_file = "[global]
        fsid = ${ceph_fsid}
        mon_host = ${ceph_mon_host}\n"
        $ceph_keyring_file = "[client.openstack]
        key = ${ceph_openstack_key}\n"
        file { "${ceph_file_name_base}.conf":
          ensure  => 'present',
          content => $ceph_conf_file,
        }

        file { "${ceph_file_name_base}.client.openstack.keyring":
          ensure  => 'present',
          content => $ceph_keyring_file,
        }
      }
    }

    if $compute_is_cephosd and $ceph_osd_initial_setup{
      $block_devices.each |Integer $index, String $value| {
        ceph::osd { "/dev/${value}":
          dmcrypt => false,
          require => Class['::ceph::profile::params'],
        }
      }
    }

    if $use_ceph_if_available or $compute_is_cephosd{
      if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat' or $openstack_release == 'caracal'){
        class { '::nova::compute::rbd':
          libvirt_rbd_user        => 'openstack',
          libvirt_rbd_secret_uuid => $ceph_libvirtuuid,
          libvirt_rbd_secret_key  => $ceph_openstack_key,
          libvirt_images_rbd_pool => 'nova',
          rbd_keyring             => 'client.openstack',
          # ceph packaging is already managed by puppet-ceph
          manage_ceph_client      => false,
          require => Ceph::Key['client.openstack'],
        }
      # rbd_keyring is deprecated on Dalmatian, and removed on Flamingo and up.
      }else{
        class { '::nova::compute::rbd':
          libvirt_rbd_user        => 'openstack',
          libvirt_rbd_secret_uuid => $ceph_libvirtuuid,
          libvirt_rbd_secret_key  => $ceph_openstack_key,
          libvirt_images_rbd_pool => 'nova',
          # ceph packaging is already managed by puppet-ceph
          manage_ceph_client      => false,
          require => Ceph::Key['client.openstack'],
        }
      }
      if $ceph_multiple_cluster_az and $ceph_multiple_az_libvirtdata {
          # If multiple ceph clusters are present, all secrets are required in libvirt
          $ceph_multiple_az_libvirtdata.each |Integer $id, String $ceph_az_libvirtdata| {
            $arr = split($ceph_az_libvirtdata, /\s+/)
            $ceph_az_libvirtuuid = $arr[0]
            $ceph_az_openstack_key = $arr[1]
            if $ceph_az_libvirtuuid != $ceph_libvirtuuid {
              #notify { "Processing uuid ${ceph_az_libvirtuuid} with key ${ceph_az_openstack_key}": }
              $rbd_keyring = "client.openstack ${ceph_az_libvirtuuid}"
              $libvirt_rbd_secret_uuid = $ceph_az_libvirtuuid

              if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat' or $openstack_release == 'caracal') {
                file { "/etc/nova/${ceph_az_libvirtuuid}-secret.xml":
                  content => template("nova/secret.xml-compute.erb"),
                  require => Ceph::Key['client.openstack'],
                }
              }else{
                # Starting with Dalmatian, we're using a .epp instead of .erb.
                file { "/etc/nova/${ceph_az_libvirtuuid}-secret.xml":
                  ensure  => 'present',
                  owner   => 'root',
                  group   => 'root',
                  mode    => '0600',
                  content => epp('nova/libvirt-secret-ceph.xml.epp', {
                    'secret_name' => $rbd_keyring,
                    'uuid'        => $libvirt_rbd_secret_uuid,
                  }),
                }
              }

              $cm = "/usr/bin/virsh secret-define --file /etc/nova/${ceph_az_libvirtuuid}-secret.xml"
              exec { "get-or-set ${ceph_az_libvirtuuid} virsh secret":
                command => $cm,
                unless  => "/usr/bin/virsh secret-list | grep -i ${ceph_az_libvirtuuid}",
                require => File["/etc/nova/${ceph_az_libvirtuuid}-secret.xml"],
              }
              $cm2 = "/usr/bin/virsh secret-set-value --secret ${ceph_az_libvirtuuid} ${ceph_az_openstack_key}"
              exec { "get-or-set ${ceph_az_libvirtuuid} virsh secret value":
                command => $cm2,
                unless  => "/usr/bin/virsh secret-get-value ${ceph_az_libvirtuuid} 2>/dev/null | grep -i ${ceph_az_openstack_key}",
                require => Exec["get-or-set ${ceph_az_libvirtuuid} virsh secret"],
              }
              Service<| title == "libvirt" |> -> Exec["get-or-set ${ceph_az_libvirtuuid} virsh secret"] -> Exec["get-or-set ${ceph_az_libvirtuuid} virsh secret value"]
            }
        }
      }
    }else{
      # If we have Ceph OSDs but we don't use /var/lib/nova/instances,
      # we still need the secret to be defined on libvirt: let's do it
      # without ::nova::compute::rbd, and by the way, let's disable
      # RBD on nova.conf, just in case someone changed his mind, and
      # disabled such config on the node.

      $rbd_keyring = 'client.openstack'
      $libvirt_rbd_secret_uuid = $ceph_libvirtuuid
      if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat' or $openstack_release == 'caracal') {
        file { '/etc/nova/secret.xml':
          content => template('nova/secret.xml-compute.erb'),
          require => Ceph::Key['client.openstack'],
        }
      }else{
        # Starting with Dalmatian, we're using a .epp instead of .erb.
        file { "/etc/nova/secret.xml":
          ensure  => 'present',
          owner   => 'root',
          group   => 'root',
          mode    => '0600',
          content => epp('nova/libvirt-secret-ceph.xml.epp', {
            'secret_name' => $rbd_keyring,
            'uuid'        => $libvirt_rbd_secret_uuid,
          }),
        }
      }

      #Variable name shrunk in favor of removing
      #the more than 140 chars puppet-lint warning.
      #variable used in the get-or-set virsh secret
      #resource.
      $cm = '/usr/bin/virsh secret-define --file /etc/nova/secret.xml | /usr/bin/awk \'{print $2}\' | sed \'/^$/d\' > /etc/nova/virsh.secret'
      exec { 'get-or-set virsh secret':
        command => $cm,
        unless  => "/usr/bin/virsh secret-list | grep -i ${libvirt_rbd_secret_uuid}",
        require => File['/etc/nova/secret.xml'],
      }
      $cm2 = "/usr/bin/virsh secret-set-value --secret ${libvirt_rbd_secret_uuid} ${ceph_openstack_key}"
      exec { 'get-or-set virsh secret value':
        command => $cm2,
        unless  => "/usr/bin/virsh secret-get-value ${libvirt_rbd_secret_uuid} 2>/dev/null | grep -i ${ceph_openstack_key}",
        require => Exec['get-or-set virsh secret'],
      }
      Service<| title == 'libvirt' |> -> Exec['get-or-set virsh secret'] -> Exec['get-or-set virsh secret value']
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        nova_config {
          'libvirt/images_type':          ensure => absent;
        }
      }
      nova_config {
        'libvirt/images_rbd_pool':      ensure => absent;
        'libvirt/images_rbd_ceph_conf': ensure => absent;
        'libvirt/rbd_secret_uuid':      value => $ceph_libvirtuuid;
        'libvirt/rbd_user':             value => 'openstack';
      }
      # make sure ceph pool exists before running nova-compute
      # Note: cannot be done on the compute nodes, since we aren't doing pools here
      # but on the controller.
      #Exec['create-nova'] -> Service['nova-compute']
    }
  }
  logrotate::rule { 'nova-common':
    path          => '/var/log/nova/*.log',
    rotate_every  => $logrotate_period,
    missingok     => true,
    compress      => true,
    delaycompress => true,
    ifempty       => false,
    copytruncate  => true,
    rotate        => '4',
  }

  ###############
  ### Neutron ###
  ###############
  if $disable_notifications {
    $neutron_notif_transport_url = undef
  } else {
    $neutron_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'neutron',
                                    'password'  => $pass_neutron_messaging,
                                  })
  }

  if $neutron_use_dvr {
    $l3_agent_mode              = 'dvr'
    $enable_distributed_routing = true
    $install_l3_agent           = true
  }else{
    $l3_agent_mode              = 'legacy'
    $enable_distributed_routing = false
    $install_l3_agent           = false
  }

  $bgp_dr_plugin = $use_dynamic_routing ? {
    true    => 'bgp',
    default => undef,
  }
  if $install_telemetry {
    $metering_plugin = 'metering'
  }else{
    $metering_plugin = undef
  }
  $plugins_list = delete_undef_values(['router', $metering_plugin, 'qos', 'trunk', 'segments', $bgp_dr_plugin])
  if $neutron_install_dragent {
    class { '::neutron::agents::bgp_dragent':
      bgp_router_id => $machine_ip,
    }
  }else{
    package { 'neutron dynamic routing':
      name   => 'neutron-bgp-dragent',
      ensure => purged,
    }
    if $use_dynamic_routing {
      package { 'neutron-dynamic-routing-common':
        name   => 'neutron-dynamic-routing-common',
        ensure => present,
      }
    }else{
      package { 'python3-neutron-dynamic-routing':
        name   => 'python3-neutron-dynamic-routing',
        ensure => purged,
      }
    }
  }

  if $install_designate {
    $notif_topics_neutron = ['notifications','notifications_designate']
  } else {
    $notif_topics_neutron = ['notifications']
  }

  class { '::neutron::logging':
    debug                         => $neutron_debug_log,
    use_json                      => $use_json_log,
    logging_context_format_string => $logging_context_format_string,
    logging_default_format_string => $logging_default_format_string,
  }
  if $openstack_release == 'rocky'{
    class { '::neutron':
      debug                      => true,
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url  => $neutron_notif_transport_url,
      rabbit_use_ssl              => $use_ssl,
      rpc_response_timeout        => $rabbitmq_message_timeout,
      amqp_durable_queues         => false,
      amqp_auto_delete            => false,
      rabbit_ha_queues            => true,
      kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
      allow_overlapping_ips       => true,
      core_plugin                 => 'ml2',
      service_plugins             => $plugins_list,
      bind_host                   => $machine_ip,
      notification_driver         => $notification_driver,
      global_physnet_mtu          => $neutron_global_physnet_mtu,
      dns_domain                  => $neutron_dns_domain,
      notification_topics         => $notif_topics_neutron,
      rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
    }
  }elsif $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' {
    class { '::neutron':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url  => $neutron_notif_transport_url,
      rabbit_use_ssl              => $use_ssl,
      rpc_response_timeout        => $rabbitmq_message_timeout,
      amqp_durable_queues         => false,
      amqp_auto_delete            => false,
      rabbit_ha_queues            => true,
      kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
      allow_overlapping_ips       => true,
      core_plugin                 => 'ml2',
      service_plugins             => $plugins_list,
      bind_host                   => $machine_ip,
      notification_driver         => $notification_driver,
      global_physnet_mtu          => $neutron_global_physnet_mtu,
      dns_domain                  => $neutron_dns_domain,
      notification_topics         => $notif_topics_neutron,
      rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
    }
  }elsif($openstack_release == 'zed' or $openstack_release == 'antelope'){
    class { '::neutron':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url  => $neutron_notif_transport_url,
      rabbit_use_ssl              => $use_ssl,
      rpc_response_timeout        => $rabbitmq_message_timeout,
      amqp_durable_queues         => false,
      amqp_auto_delete            => false,
      rabbit_ha_queues            => true,
      kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
      core_plugin                 => 'ml2',
      service_plugins             => $plugins_list,
      bind_host                   => $machine_ip,
      notification_driver         => $notification_driver,
      global_physnet_mtu          => $neutron_global_physnet_mtu,
      dns_domain                  => $neutron_dns_domain,
      notification_topics         => $notif_topics_neutron,
      rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
    }
  }elsif($openstack_release == 'bobcat'){
    class { '::neutron':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url  => $neutron_notif_transport_url,
      rabbit_use_ssl              => $use_ssl,
      rpc_response_timeout        => $rabbitmq_message_timeout,
      amqp_durable_queues         => $amqp_durable_queues,
      amqp_auto_delete            => false,
      rabbit_ha_queues            => $rabbit_ha_queues,
      rabbit_quorum_queue         => $rabbit_quorum_queue,
      kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
      core_plugin                 => 'ml2',
      service_plugins             => $plugins_list,
      bind_host                   => $machine_ip,
      notification_driver         => $notification_driver,
      global_physnet_mtu          => $neutron_global_physnet_mtu,
      dns_domain                  => $neutron_dns_domain,
      notification_topics         => $notif_topics_neutron,
      rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
    }
  # Caracal and up.
  }else{
    class { '::neutron':
      default_transport_url         => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url    => $neutron_notif_transport_url,
      rabbit_use_ssl                => $use_ssl,
      rpc_response_timeout          => $rabbitmq_message_timeout,
      amqp_durable_queues           => $amqp_durable_queues,
      amqp_auto_delete              => false,
      rabbit_ha_queues              => $rabbit_ha_queues,
      rabbit_quorum_queue           => $rabbit_quorum_queue,
      rabbit_transient_quorum_queue => $rabbit_transient_quorum_queue,
      rabbit_use_queue_manager      => $use_queue_manager,
      rabbit_stream_fanout          => $rabbit_stream_fanout,
      rabbit_qos_prefetch_count     => $rabbit_qos_prefetch_count,
      kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
      core_plugin                   => 'ml2',
      service_plugins               => $plugins_list,
      bind_host                     => $machine_ip,
      notification_driver           => $notification_driver,
      global_physnet_mtu            => $neutron_global_physnet_mtu,
      dns_domain                    => $neutron_dns_domain,
      notification_topics           => $notif_topics_neutron,
      rabbit_transient_queues_ttl   => $rabbitmq_transient_queue_expire,
    }
  }
  neutron_config {
    'nova/cafile':         value => $api_endpoint_ca_file;
    'database/connection': value => $facts['os_service_default'];
  }
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'{
    class { '::neutron::server::notifications':
      auth_url => $keystone_admin_uri,
      username => $nova_user,
      password => $pass_nova_authtoken,
    }
  # Victoria and up
  }else{
    class { 'neutron::server::notifications::nova':
      auth_url    => $keystone_admin_uri,
      username    => $nova_user,
      password    => $pass_nova_authtoken,
      region_name => $region_name,
    }
  }
  if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
    class { '::neutron::client': }
  }
  class { '::neutron::keystone::authtoken':
    username             => $neutron_user,
    password             => $pass_neutron_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    cafile               => $api_endpoint_ca_file,
    region_name          => $region_name,
  }
  if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat'){
    class { '::neutron::agents::ml2::ovs':
      local_ip                   => $vmnet_ip,
      tunnel_types               => ['vxlan'],
      bridge_uplinks             => ['eth0'],
      bridge_mappings            => $bridge_mapping_list,
      extensions                 => 'qos',
      l2_population              => true,
      arp_responder              => true,
      firewall_driver            => 'iptables_hybrid',
      drop_flows_on_start        => false,
      enable_distributed_routing => $enable_distributed_routing,
      manage_vswitch             => false,
    }
  # Caracal and up
  } else {
    class { '::neutron::agents::ml2::ovs':
      local_ip                   => $vmnet_ip,
      tunnel_types               => ['vxlan'],
      bridge_uplinks             => ['eth0'],
      bridge_mappings            => $bridge_mapping_list,
      extensions                 => 'qos',
      l2_population              => true,
      arp_responder              => true,
      firewall_driver            => 'openvswitch',
      drop_flows_on_start        => false,
      enable_distributed_routing => $enable_distributed_routing,
      manage_vswitch             => false,
    }
  }
  if $install_l3_agent {
    class { '::neutron::agents::l3':
      interface_driver      => 'openvswitch',
      debug                 => true,
      agent_mode            => $l3_agent_mode,
      ha_enabled            => false,
      extensions            => 'port_forwarding,fip_qos,gateway_ip_qos',
      ha_vrrp_auth_type     => 'PASS',
      ha_vrrp_auth_password => $pass_neutron_vrrpauth,
    }
    ~>systemd::unit_file { 'keepalived.service':
      enable => 'mask',
      active => false,
    }
    neutron_l3_agent_config {
      'DEFAULT/external_network_bridge': value => '';
    }
  } else {
    package { 'neutron-l3-agent':
      ensure => absent,
    }
  }

  if $has_subrole_designate {
    $extension_drivers = 'port_security,qos,dns,dns_domain_ports'
  }else{
    $extension_drivers = 'port_security,qos'
  }

  if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
    class { '::neutron::plugins::ml2':
      type_drivers         => ['flat', 'vxlan', 'vlan', ],
      tenant_network_types => ['flat', 'vxlan', 'vlan', ],
      extension_drivers    => $extension_drivers,
      mechanism_drivers    => 'openvswitch,l2population',
      firewall_driver      => 'iptables_v2',
      flat_networks        => $external_network_list,
      network_vlan_ranges  => $external_network_list,
      vni_ranges           => "${neutron_vxlan_vni_min}:${neutron_vxlan_vni_max}",
      path_mtu             => $neutron_path_mtu,
    }
  }else{
    class { '::neutron::plugins::ml2':
      type_drivers         => ['flat', 'vxlan', 'vlan', ],
      tenant_network_types => ['flat', 'vxlan', 'vlan', ],
      extension_drivers    => $extension_drivers,
      mechanism_drivers    => 'openvswitch,l2population',
      flat_networks        => $external_network_list,
      network_vlan_ranges  => $external_network_list,
      vni_ranges           => "${neutron_vxlan_vni_min}:${neutron_vxlan_vni_max}",
      path_mtu             => $neutron_path_mtu,
    }
  }
  if $has_subrole_dhcp_agent {
    class { '::neutron::agents::dhcp':
      interface_driver         => 'openvswitch',
      debug                    => true,
      enable_isolated_metadata => true,
      enable_metadata_network  => true,
      dnsmasq_dns_servers      => split($neutron_dnsmasq_dns_servers, ';'),
    }
  }else{
    package { 'neutron-dhcp-agent':
      ensure => absent,
    }
  }
  if $install_telemetry {
    class { '::neutron::agents::metering':
      interface_driver => 'openvswitch',
      driver           => 'iptables',
      debug            => true,
    }
  }else{
    package { 'neutron-metering-agent':
      ensure => purged,
    }
  }
  if $neutron_use_dvr {
    class { '::neutron::agents::metadata':
      debug             => true,
      shared_secret     => $pass_metadata_proxy_shared_secret,
      metadata_workers  => 2,
      metadata_protocol => 'http',
      metadata_host     => 'localhost',
    }
  } else {
    package { 'remove-metadata-agent-in-non-dvr':
      name   => 'neutron-metadata-agent',
      ensure => purged,
    }
  }
  package { 'l2gw networking':
    name => 'python3-networking-l2gw',
    ensure => installed,
  }
  logrotate::rule { 'neutron-common':
    path          => '/var/log/neutron/*.log',
    rotate_every  => $logrotate_period,
    missingok     => true,
    compress      => true,
    delaycompress => true,
    ifempty       => false,
    copytruncate  => true,
    rotate        => '4',
  }
  ##############
  ### Cinder ###
  ##############
  # If using Ceph, then we setup cinder-volume and cinder-backup over Ceph
  # on each and every compute nodes. The goal is to spread the load away
  # from the controller nodes.
  if $cluster_has_osds {
    include ::cinder::client

    if $disable_notifications {
      $cinder_notif_transport_url = undef
    } else {
      $cinder_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'cinder',
                                      'password'  => $pass_cinder_messaging,
                                    })
    }

    # Cinder main class (ie: cinder-common config)
    if $openstack_release == 'rocky'{
      class { '::cinder':
        debug                 => true,
        default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'cinder',
          'password'  => $pass_cinder_messaging,
        }),
        database_connection         => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
        rabbit_use_ssl              => $use_ssl,
        rpc_response_timeout        => $rabbitmq_message_timeout,
        amqp_durable_queues         => false,
        amqp_auto_delete            => false,
        rabbit_ha_queues            => true,
        kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
        storage_availability_zone   => $availability_zone,
        rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
      }
    }else{
      if($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection         => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
          rabbit_use_ssl              => $use_ssl,
          rpc_response_timeout        => $rabbitmq_message_timeout,
          amqp_durable_queues         => false,
          amqp_auto_delete            => false,
          rabbit_ha_queues            => true,
          kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
          storage_availability_zone   => $availability_zone,
          rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
        }
        class { 'cinder::ceilometer':
          notification_transport_url => $cinder_notif_transport_url,
          notification_driver        => $notification_driver,
        }
      }else{
        if $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
          class { '::cinder':
            default_transport_url => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'cinder',
              'password'  => $pass_cinder_messaging,
            }),
            database_connection         => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
            rabbit_use_ssl              => $use_ssl,
            rpc_response_timeout        => $rabbitmq_message_timeout,
            amqp_durable_queues         => false,
            amqp_auto_delete            => false,
            rabbit_ha_queues            => true,
            kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
            notification_transport_url  => $cinder_notif_transport_url,
            notification_driver         => $notification_driver,
            storage_availability_zone   => $availability_zone,
            rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
          }
        # Yoga and up
        }else{
          class { '::cinder::db':
            database_connection              => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
            database_connection_recycle_time => 1800,
          }
          if($openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
            class { '::cinder':
              default_transport_url => os_transport_url({
                'transport' => $messaging_default_proto,
                'hosts'     => fqdn_rotate($all_masters),
                'port'      => $messaging_default_port,
                'username'  => 'cinder',
                'password'  => $pass_cinder_messaging,
              }),
              rabbit_use_ssl              => $use_ssl,
              rpc_response_timeout        => $rabbitmq_message_timeout,
              amqp_durable_queues         => false,
              amqp_auto_delete            => false,
              rabbit_ha_queues            => true,
              kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
              notification_transport_url  => $cinder_notif_transport_url,
              notification_driver         => $notification_driver,
              storage_availability_zone   => $availability_zone,
              rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
            }
          }elsif($openstack_release == 'bobcat'){
            class { '::cinder':
              default_transport_url => os_transport_url({
                'transport' => $messaging_default_proto,
                'hosts'     => fqdn_rotate($all_masters),
                'port'      => $messaging_default_port,
                'username'  => 'cinder',
                'password'  => $pass_cinder_messaging,
              }),
              rabbit_use_ssl              => $use_ssl,
              rpc_response_timeout        => $rabbitmq_message_timeout,
              amqp_durable_queues         => false,
              amqp_auto_delete            => false,
              rabbit_ha_queues            => $rabbit_ha_queues,
              rabbit_quorum_queue         => $rabbit_quorum_queue,
              kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
              notification_transport_url  => $cinder_notif_transport_url,
              notification_driver         => $notification_driver,
              storage_availability_zone   => $availability_zone,
              rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
            }
          # Caracal and up.
          }else{
            class { '::cinder':
              default_transport_url => os_transport_url({
                'transport' => $messaging_default_proto,
                'hosts'     => fqdn_rotate($all_masters),
                'port'      => $messaging_default_port,
                'username'  => 'cinder',
                'password'  => $pass_cinder_messaging,
              }),
              rabbit_use_ssl                => $use_ssl,
              rpc_response_timeout          => $rabbitmq_message_timeout,
              amqp_durable_queues           => false,
              amqp_auto_delete              => false,
              rabbit_ha_queues              => $rabbit_ha_queues,
              rabbit_quorum_queue           => $rabbit_quorum_queue,
              rabbit_transient_quorum_queue => $rabbit_transient_quorum_queue,
              rabbit_use_queue_manager      => $use_queue_manager,
              rabbit_stream_fanout          => $rabbit_stream_fanout,
              rabbit_qos_prefetch_count     => $rabbit_qos_prefetch_count,
              kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
              notification_transport_url    => $cinder_notif_transport_url,
              notification_driver           => $notification_driver,
              storage_availability_zone     => $availability_zone,
              rabbit_transient_queues_ttl   => $rabbitmq_transient_queue_expire,
            }
          }
        }
      }
      class { '::cinder::logging':
        debug                         => $cinder_debug_log,
        use_json                      => $use_json_log,
        logging_context_format_string => $logging_context_format_string,
        logging_default_format_string => $logging_default_format_string,
      }
    }

    if $extswift_use_external and $cinder_backup_use_external_swift{
      $backup_backend_real         = 'cinder.backup.drivers.swift.SwiftBackupDriver'
      $backup_swift_auth_url       = $extswift_auth_url
      $backup_swift_url            = $extswift_proxy_url
      $backup_swift_user_domain    = $extswift_user_domain_name
      $backup_swift_project_domain = $extswift_project_domain_name
      $backup_swift_project        = $extswift_project_name
      $backup_swift_user           = $extswift_user_name
      $backup_swift_key            = $extswift_password
      $backup_swift_auth_type      = 'single_user'
    }else{
      $backup_swift_auth_url       = $keystone_auth_uri
      $backup_swift_url            = undef
      $backup_swift_user_domain    = undef
      $backup_swift_project_domain = undef
      $backup_swift_project        = undef
      $backup_swift_user           = undef
      $backup_swift_key            = undef
      $backup_swift_auth_type      = 'per_user'
      case $backup_backend {
        'ceph': {
          $backup_backend_real = 'cinder.backup.drivers.ceph.CephBackupDriver'
        }
        'swift': {
          $backup_backend_real = 'cinder.backup.drivers.swift.SwiftBackupDriver'
        }
        'none': {
          $backup_backend_real = ''
        }
      }
    }

    cinder_config {
      'DEFAULT/backup_driver':               value => $backup_backend_real;
      'DEFAULT/snapshot_clone_size':         value => '200';
      'DEFAULT/rbd_pool':                    value => 'cinder';
      'DEFAULT/rbd_ceph_conf':               value => '/etc/ceph/ceph.conf';
      'DEFAULT/rbd_secret_uuid':             value => $ceph_libvirtuuid;
      'DEFAULT/backup_swift_auth_url':       value => $backup_swift_auth_url;
      'DEFAULT/backup_swift_url':            value => $backup_swift_url;
      'DEFAULT/backup_swift_auth':           value => $backup_swift_auth_type;
      'DEFAULT/backup_swift_user_domain':    value => $backup_swift_user_domain;
      'DEFAULT/backup_swift_project_domain': value => $backup_swift_project_domain;
      'DEFAULT/backup_swift_project':        value => $backup_swift_project;
      'DEFAULT/backup_swift_user':           value => $backup_swift_user;
      'DEFAULT/backup_swift_key':            value => $backup_swift_key;
      'DEFAULT/backup_swift_auth_version':   value => '3';
      'DEFAULT/backup_swift_object_size':    value => 1048576000;
      'DEFAULT/keystone_catalog_info':       value => 'identity:keystone:publicURL';
      'DEFAULT/backup_ceph_user':            value => 'openstack';
      'DEFAULT/backup_ceph_pool':            value => 'cinderback';
      'DEFAULT/backup_swift_ca_cert_file':   value => $api_endpoint_ca_file;
      'ssl/ca_file':                         value => $api_endpoint_ca_file;
    }

    # Configure the authtoken
    class { '::cinder::keystone::authtoken':
      username             => $cinder_user,
      password             => $pass_cinder_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }

    # This is mandatory starting with bobcat, and this is supported in puppet-cinder only in train and up.
    # This is related to CVE-2023-2088.
    if ($openstack_release != 'rocky' and $openstack_release != 'stein'){
      class { '::cinder::keystone::service_user':
        username                => $cinder_user,
        password                => $pass_cinder_authtoken,
        auth_url                => $keystone_admin_uri,
        project_name            => 'services',
        region_name             => $region_name,
        send_service_user_token => true,
        cafile                  => $api_endpoint_ca_file,
      }
    }

    if $ceph_multiple_cluster_az {
      $volume_cluster = "ceph_${ceph_az}"
    }else{
      $volume_cluster = "ceph_cluster"
    }

    if $ceph_multiple_cluster_az and $ceph_multiple_az_cinder_conf {
      # Map the ceph_multiple_az_cinder_conf to create backend names and configure cinder::backend::rbd resources
      $cinder_ceph_backend_names = $ceph_multiple_az_cinder_conf.map |Hash $conf| {
        $ceph_az_availability_zone_id = $conf['az']
        $ceph_az_availability_zone_name = $conf['az_name']
        $ceph_az_libvirtuuid = $conf['libvirtuuid']

        $cinder_ceph_backend_az_name = "CEPH_${ceph_az_availability_zone_id}"

        cinder::backend::rbd { $cinder_ceph_backend_az_name:
          rbd_user                  => 'openstack',
          rbd_pool                  => 'cinder',
          rbd_secret_uuid           => $ceph_az_libvirtuuid,
          rbd_ceph_conf             => "/etc/ceph/ceph-az${ceph_az_availability_zone_id}.conf",
          backend_availability_zone => $ceph_az_availability_zone_name,
          manage_volume_type        => false,
          backend_host              => $machine_hostname,
          volume_backend_name       => 'CEPH_1',
        }

        $cinder_ceph_backend_az_name
      }

      class { '::cinder::backends':
        enabled_backends => $cinder_ceph_backend_names,
      }
    } else {
      $cinder_ceph_backend_name = 'CEPH_1'

      class { '::cinder::backends':
        enabled_backends => [$cinder_ceph_backend_name],
      }

      cinder::backend::rbd { $cinder_ceph_backend_name:
        rbd_user           => 'openstack',
        rbd_pool           => 'cinder',
        rbd_secret_uuid    => $ceph_libvirtuuid,
        manage_volume_type => false,
        backend_host       => $machine_hostname,
      }
    }

    # Clear volumes on delete (for data security)
    class { '::cinder::volume':
      volume_clear => 'zero',
      # This is needed, so that volumes aren't bound to a single
      # cinder-volume service in Ceph mode (ie: a volume can be
      # managed by any volume node).
      cluster      => $volume_cluster,
    }
    $zookeeper_ensemble_string = join($all_masters_ip,':2181,')
    # This makes it so that if a cinder-volume service dies, the global cinder-volume
    # service continue to work.
    class { '::cinder::coordination':
      backend_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
    }

    # A cinder-backup service on each volume nodes
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria'){
        class { '::cinder::backup': }
        cinder_config {
          'DEFAULT/backup_workers': value => '4';
        }
    }else{
        class { '::cinder::backup':
            backup_workers => 4,
        }
    }

    # Avoids Cinder to lookup for the catalogue
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat' or $openstack_release == 'caracal' or $openstack_release == 'dalmatian' or $openstack_release == 'epoxy'){
      class { '::cinder::glance':
        glance_api_servers => "${proto}://${base_url_glance}",
      }
    }else{
      # Flamingo and up.
      class { '::cinder::glance':
        username           => $cinder_user,
        password           => $pass_cinder_authtoken,
        cafile             => $api_endpoint_ca_file,
        glance_api_servers => "${proto}://${base_url_glance}",
        auth_type          => 'password',
        auth_url           => $keystone_admin_uri,
      }
    }
    class { '::cinder::nova':
      username    => $nova_user,
      password    => $pass_nova_authtoken,
      auth_url    => $keystone_admin_uri,
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      auth_type   => 'password',
    }
    logrotate::rule { 'cinder-common':
      path          => '/var/log/cinder/*.log',
      rotate_every  => $logrotate_period,
      missingok     => true,
      compress      => true,
      delaycompress => true,
      ifempty       => false,
      copytruncate  => true,
      rotate        => '4',
    }
  }

  ##################
  ### Ceilometer ###
  ##################
  if $has_subrole_ceilometer and $install_telemetry{
    if $openstack_release == 'rocky'{
      class { '::ceilometer':
        debug                      => true,
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl                => $use_ssl,
        kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
        rpc_response_timeout          => $rabbitmq_message_timeout,
        amqp_durable_queues           => $amqp_durable_queues,
        amqp_auto_delete              => false,
        rabbit_ha_queues              => $rabbit_ha_queues,
        rabbit_transient_queues_ttl   => $rabbitmq_transient_queue_expire,
      }
    }else{
      if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat'){
        class { '::ceilometer':
          telemetry_secret           => $pass_ceilometer_telemetry,
          default_transport_url      => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          notification_transport_url => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($notif_rabbit_servers),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          rabbit_use_ssl                => $use_ssl,
          kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
          rpc_response_timeout          => $rabbitmq_message_timeout,
          amqp_durable_queues           => $amqp_durable_queues,
          amqp_auto_delete              => false,
          rabbit_ha_queues              => $rabbit_ha_queues,
          rabbit_transient_queues_ttl   => $rabbitmq_transient_queue_expire,
        }
      # Caracal and up.
      }else{
        class { '::ceilometer':
          telemetry_secret           => $pass_ceilometer_telemetry,
          default_transport_url      => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          notification_transport_url => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($notif_rabbit_servers),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          rabbit_use_ssl                => $use_ssl,
          kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
          rpc_response_timeout          => $rabbitmq_message_timeout,
          amqp_durable_queues           => $amqp_durable_queues,
          amqp_auto_delete              => false,
          rabbit_ha_queues              => $rabbit_ha_queues,
          rabbit_quorum_queue           => $rabbit_quorum_queue,
          rabbit_transient_quorum_queue => $rabbit_transient_quorum_queue,
          rabbit_use_queue_manager      => $use_queue_manager,
          rabbit_stream_fanout          => $rabbit_stream_fanout,
          rabbit_qos_prefetch_count     => $rabbit_qos_prefetch_count,
          rabbit_transient_queues_ttl   => $rabbitmq_transient_queue_expire,
        }
      }
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::ceilometer::keystone::authtoken':
        username             => $ceilometer_user,
        password             => $pass_ceilometer_authtoken,
        auth_url             => $keystone_admin_uri,
        www_authenticate_uri => $keystone_auth_uri,
        cafile               => $api_endpoint_ca_file,
        region               => $region_name,
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria'{
      class { '::ceilometer::agent::auth':
        auth_user     => $ceilometer_user,
        auth_password => $pass_ceilometer_authtoken,
        auth_url      => $keystone_auth_uri,
        auth_cacert   => $api_endpoint_ca_file,
        auth_region   => $region_name,
      }
    }else{
      class { '::ceilometer::agent::service_credentials':
        username    => $ceilometer_user,
        password    => $pass_ceilometer_authtoken,
        auth_url    => $keystone_auth_uri,
        cafile      => $api_endpoint_ca_file,
        region_name => $region_name,
      }
    }
    # Looks like the above sets ca_file instead of cafile.
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      ceilometer_config {
        'service_credentials/cafile': value => $api_endpoint_ca_file;
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      class { '::ceilometer::agent::compute':
        instance_discovery_method => 'libvirt_metadata',
      }
    }else{
        $standard_pollsters_array =
                [ { 'name'     => 'nova_poll_cpu_memory',
                    'interval' => '300',
                    'meters'   => [ 'cpu', 'vcpus', 'memory.usage' ]},
                  { 'name'     => 'nova_poll_network',
                    'interval' => '300',
                    'meters'   => [ 'network.incoming.bytes', 'network.incoming.packets', 'network.outgoing.bytes', 'network.outgoing.packets' ]},
                  { 'name'     => 'nova_poll_disk',
                    'interval' => '300',
                    'meters'   => [ 'disk.device.read.requests', 'disk.device.read.bytes', 'disk.device.write.requests', 'disk.device.write.bytes', 'disk.root.size' ]},
                  { 'name'     => 'nova_poll_metadata',
                    'interval' => '300',
                    'meters'   => [ 'ceilometer.instance.os.metadata' ]},
                ]

      $polling_config = {
        'sources' => $standard_pollsters_array
      }

      class { '::ceilometer::agent::polling':
        central_namespace         => false,
        compute_namespace         => true,
        ipmi_namespace            => false,
        instance_discovery_method => 'libvirt_metadata',
        manage_polling            => true,
        polling_config            => $polling_config,
        polling_interval          => 300,
      }
    }
    package { 'ceilometer-instance-poller':
      ensure => installed,
    }
    class { '::ceilometer::logging':
      debug                         => $ceilometer_debug_log,
      use_json                      => $use_json_log,
      logging_context_format_string => $logging_context_format_string,
      logging_default_format_string => $logging_default_format_string,
    }
    logrotate::rule { 'ceilometer-common':
      path          => '/var/log/ceilometer/*.log',
      rotate_every  => $logrotate_period,
      missingok     => true,
      compress      => true,
      delaycompress => true,
      ifempty       => false,
      copytruncate  => true,
      rotate        => '4',
    }
  } else {
    package { 'ceilometer-instance-poller':
      ensure => purged,
    }
    package { 'python3-ceilometer':
      ensure => purged,
    }
  }
  ############################
  ### Install Manila-share ###
  ############################
  # Note: manila-share has to be setup in a compute node, because
  # it needs access to br-int and OVS.
  # It currently doesn't support HA, so it must be installed in a single
  # compute node only.
  if $has_subrole_manila and $manila_share_install and $install_manila{
    if $disable_notifications {
      $manila_notif_transport_url = undef
    }else{
      $manila_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'manila',
                                      'password'  => $pass_manila_messaging,
                                    })
    }
    class { '::manila::db':
      database_connection   => "mysql+pymysql://manila:${pass_manila_db}@${sql_vip_ip}/maniladb?charset=utf8",
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
      class { '::manila':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'manila',
          'password'  => $pass_manila_messaging,
        }),
        notification_transport_url  => $manila_notif_transport_url,
        notification_driver         => $notification_driver,
        rabbit_use_ssl              => $use_ssl,
        amqp_durable_queues         => false,
        amqp_auto_delete            => false,
        rabbit_ha_queues            => true,
        kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
        host                        => $machine_hostname,
        rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
      }
    }elsif($openstack_release == 'bobcat'){
      class { '::manila':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'manila',
          'password'  => $pass_manila_messaging,
        }),
        notification_transport_url  => $manila_notif_transport_url,
        notification_driver         => $notification_driver,
        rabbit_use_ssl              => $use_ssl,
        amqp_durable_queues         => $amqp_durable_queues,
        amqp_auto_delete            => false,
        rabbit_ha_queues            => $rabbit_ha_queues,
        rabbit_quorum_queue         => $rabbit_quorum_queue,
        kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
        host                        => $machine_hostname,
        rabbit_transient_queues_ttl => $rabbitmq_transient_queue_expire,
      }
    # Caracal and up.
    }else{
      class { '::manila':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'manila',
          'password'  => $pass_manila_messaging,
        }),
        notification_transport_url    => $manila_notif_transport_url,
        notification_driver           => $notification_driver,
        rabbit_use_ssl                => $use_ssl,
        amqp_durable_queues           => $amqp_durable_queues,
        amqp_auto_delete              => false,
        rabbit_ha_queues              => $rabbit_ha_queues,
        rabbit_quorum_queue           => $rabbit_quorum_queue,
        rabbit_transient_quorum_queue => $rabbit_transient_quorum_queue,
        rabbit_use_queue_manager      => $use_queue_manager,
        rabbit_stream_fanout          => $rabbit_stream_fanout,
        rabbit_qos_prefetch_count     => $rabbit_qos_prefetch_count,
        kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
        host                          => $machine_hostname,
        rabbit_transient_queues_ttl   => $rabbitmq_transient_queue_expire,
      }
    }
    # This *must* be called before ::manila::api that contains an include
    class { '::manila::keystone::authtoken':
      username             => $manila_user,
      password             => $pass_manila_authtoken,
      user_domain_name     => 'Default',
      project_domain_name  => 'Default',
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::manila::compute::nova':
      cafile      => $api_endpoint_ca_file,
      username    => $nova_user,
      password    => $pass_nova_authtoken,
      region_name => $region_name,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::image::glance':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => $glance_user,
      password    => $pass_glance_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::network::neutron':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => $neutron_user,
      password    => $pass_neutron_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::volume::cinder':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => $cinder_user,
      password    => $pass_cinder_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::share': }
    # manila-share cannot be setup on the controllers,
    # because it needs access to OVS.
    #class { '::manila::share': }
    class { '::manila::backends':
      enabled_share_backends => ['generic'],
    }
    # Deploy the generic driver using driver_handles_share_servers=True (ie: in VM shares...)
    ::manila::backend::generic { 'generic':
      driver_handles_share_servers => true,
      share_backend_name           => 'GENERIC',
    }
    ::manila::service_instance { 'generic':
      service_instance_user           => 'debian',
      service_instance_password       => 'do-not-use-passwords',
      service_image_name              => 'manila-service-image',
      create_service_image            => false,
      manila_service_keypair_name     => 'manila-ssh-key',
      path_to_public_key              => '/etc/manila/.ssh/id_rsa.pub',
      path_to_private_key             => '/etc/manila/.ssh/id_rsa',
      service_instance_flavor_id      => '100',
      service_instance_security_group => 'manila-security-group',
      service_network_name            => 'manila-service-network',
      service_network_cidr            => $manila_network_cidr,
      service_network_division_mask   => $manila_network_division_mask,
    }
    manila_config {
      "DEFAULT/my_ip": value => $machine_ip;
    }
    logrotate::rule { 'manila-common':
      path          => '/var/log/manila/*.log',
      rotate_every  => $logrotate_period,
      missingok     => true,
      compress      => true,
      delaycompress => true,
      ifempty       => false,
      copytruncate  => true,
      rotate        => '4',
    }
  }

  ############################
  ### OpenVSwitch defaults ###
  ############################
  file { '/etc/default/openvswitch-switch':
    content => '# Managed by Puppet
OVS_CTL_OPTS="--delete-bridges"',
  }
}
