<?php

function get_cluster_password($con, $conf, $cluster_id, $service_type, $password_type, $multiple_ceph_as="no", $az=0){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    $q = "SELECT pass,passtxt1,passtxt2 FROM passwords WHERE cluster='$cluster_id' AND service='$service_type' AND passtype='$password_type' AND az='$az'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        # If password doesn't exist, generate it!
        insert_cluster_pass($con, $conf, $cluster_id, $service_type, $password_type, $multiple_ceph_as, $az);
        $q = "SELECT pass,passtxt1,passtxt2 FROM passwords WHERE cluster='$cluster_id' AND service='$service_type' AND passtype='$password_type' AND az='$az'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find password $password_type for service $service_type.";
            return $json;
        }
    }
    $pass_a = mysqli_fetch_array($r);
    if(($service_type == "nova" || $service_type == "keystone" || $service_type == "glance" || $service_type == "ceph" || $service_type == "bill") && $password_type == "ssh"){
        $json["data"]["ssh_pub"] = $pass_a["passtxt1"];
        $json["data"]["ssh_priv"] = $pass_a["passtxt2"];
    }else{
        $json["data"] = $pass_a["pass"];
    }
    return $json;
}

function get_ssh_keypair($con, $conf, $cluster_id, $hostname, $username){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";
    $q = "SELECT * FROM sshkeypairs WHERE clusterid='$cluster_id' AND hostname='$hostname' AND username='$username'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        # If ssh key doesn't exist, generate it!
        insert_ssh_key($con, $conf, $cluster_id, $hostname, $username);
        $q = "SELECT * FROM sshkeypairs WHERE clusterid='$cluster_id' AND hostname='$hostname' AND username='$username'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find ssh keypair for $hostname and user $username.";
            return $json;
        }
    }
    $a = mysqli_fetch_array($r);
    $json["data"]["pubkey"] = $a["pubkey"];
    $json["data"]["privatekey"] = $a["privatekey"];
    return $json;
}

function get_non_system_block_devices($con, $machine){
    $machine_id = $machine['id'];
    if($machine["install_on_raid"] == "no"){
        if($machine["dest_blk"] == "none"){
            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
        }else{
            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["dest_blk"]."'";
        }
    }else{
        $os_disks = [$machine["raid_dev0"], $machine["raid_dev1"]];
        switch($machine["raid_type"]){
            case "0":
            case "1":
                break;
            case "10":
                array_push($os_disks, $machine['raid_dev2'], $machine['raid_dev3']);
                break;
            case "5":
            default:
                die("Not supported yet.");
                break;
        }
        $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT IN ('".join("','", $os_disks)."')";
    }
    $q .= " ORDER BY name";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n < 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find block devices (SQL query: $q) line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $disks = [];
    for($i=0;$i<$n;$i++){
        $a = mysqli_fetch_array($r);
        $disks[] = $a["name"];
    }
    return $disks;
}

# $role can be either 'cephmon' or 'billmon'
function enc_get_mon_nodes($con,$conf,$cluster_id,$role='cephmon',$use_az='no',$az=0){
    $cephmon_where_clause = "";
    if($role == 'cephmon'){
        if($use_az == "yes"){
            $role_to_select = "cephmon";
            $cephmon_where_clause = "AND machines.ceph_az='$az'";
        }else{
            $q = "SELECT * FROM machines WHERE role='cephmon' AND cluster='$cluster_id'";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $n = mysqli_num_rows($r);
            if($n < 1){
                // If there's no specific cephmon/billmon nodes, then controllers will assume that role
                $q = "SELECT * FROM machines WHERE role='controller' AND cluster='$cluster_id'";
                $r = mysqli_query($con, $q);
                if($r === FALSE){
                    $json["status"] = "error";
                    $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                    return $json;
                }
                $n = mysqli_num_rows($r);
                if($n < 1){
                    $json["status"] = "error";
                    $json["message"] = "No Ceph MON or controllers in database.";
                    return $json;
                }
                $role_to_select = "controller";
            }else{
                $role_to_select = "cephmon";
            }
        }
    }else{
        $role_to_select = "billmon";
    }

    $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='$role_to_select' $cephmon_where_clause AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' AND networks.role!='vip' AND machines.cluster='$cluster_id' ORDER BY hostname ASC";

    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);

    $osd_members = "";
    $osd_ips = "";
    for($i=0;$i<$n;$i++){
        $machine = mysqli_fetch_array($r);
        $machine_hostname = $machine["hostname"];
        $machine_ipaddr   = $machine["ipaddr"];
        if($osd_members == ""){
            $osd_members = $machine_hostname;
            $osd_ips     = $machine_ipaddr;
        }else{
            $osd_members = $osd_members . "," . $machine_hostname;
            $osd_ips     = $osd_ips . "," . $machine_ipaddr;
        }
    }
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";
    $json["data"]["osd_members"] = $osd_members;
    $json["data"]["osd_ips"] = $osd_ips;
    return $json;
}

function puppet_enc($con,$conf){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    // This returns the external node classifyer yaml file, to be used by puppet.

    // Get our machine object in db, using hostname to search, as this is
    // what the enc script is given as parameter
    $safe_hostname = safe_fqdn("hostname");

    // OCI supports configuring itself, in such case,
    // We are ... we ! :)
    if($safe_hostname == gethostname()){
        $enc_file = "---\n";
        $enc_file .= "classes:\n";
        $enc_file .= "   oci::pxe_server:\n";
        $json["data"] = $enc_file;
        return $json;
    }

    $q = "SELECT * FROM machines WHERE hostname='$safe_hostname'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "No such hostname in database.";
        return $json;
    }
    $machine = mysqli_fetch_array($r);
    $machine_id   = $machine["id"];
    $machine_role = $machine["role"];
    $cluster_id   = $machine["cluster"];
    $machine_location = $machine["location_id"];
    $machine_force_no_bgp2host = $machine["force_no_bgp2host"];

    $machine_networks = slave_fetch_networks($con, $conf, $machine_id);
    if(sizeof($machine_networks["networks"]) == 0){
        $json["status"] = "error";
        $json["message"] = "Machine has no network.";
        return $json;
    }

    // Fetch matching cluster object
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cluster not found in database.";
        return $json;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_domain = $cluster["domain"];
    $cluster_name = $cluster["name"];
    $cluster_region_name = $cluster["region_name"];
    $cluster_bgp_to_the_host = $cluster["bgp_to_the_host"];
    if($cluster_bgp_to_the_host == "yes" && $machine_force_no_bgp2host == "no"){
        $machine_bgp2host = "yes";
        $machine_puppet_bgp2host = "true";
    }else{
        $machine_bgp2host = "no";
        $machine_puppet_bgp2host = "false";
    }
    $cluster_initial_cluster_setup = $cluster["initial_cluster_setup"];
    if($cluster_initial_cluster_setup == "yes"){
        $cluster_initial_cluster_setup = "true";
    }else{
        $cluster_initial_cluster_setup = "false";
    }
    $cluster_statsd_hostname = $cluster["statsd_hostname"];
    $cluster_time_server_host = $cluster["time_server_host"];
    $amp_secgroup_list        = $cluster["amp_secgroup_list"];
    $amp_boot_network_list    = $cluster["amp_boot_network_list"];
    $disable_notifications    = $cluster["disable_notifications"];
    if($disable_notifications == "yes"){
        $disable_notifications = "true";
    }else{
        $disable_notifications = "false";
    }
    $self_signed_api_cert = $cluster["self_signed_api_cert"];
    if($self_signed_api_cert == "yes"){
        $self_signed_api_cert = "true";
    }else{
        $self_signed_api_cert = "false";
    }
    $enable_monitoring_graphs = $cluster["enable_monitoring_graphs"];
    if($enable_monitoring_graphs == "yes"){
        $enable_monitoring_graphs = "true";
    }else{
        $enable_monitoring_graphs = "false";
    }
    $monitoring_graphite_host = $cluster["monitoring_graphite_host"];
    $monitoring_graphite_port = $cluster["monitoring_graphite_port"];

    $swift_object_replicator_concurrency = $cluster["swift_object_replicator_concurrency"];
    $swift_rsync_connection_limit = $cluster["swift_rsync_connection_limit"];

    $machine_networks = slave_fetch_network_config($con, $conf, $machine_id);
    if(sizeof($machine_networks["networks"]) == 0){
        $json["status"]  = "error";
        $json["message"] = "No network configured for this machine.";
        return $out;
    }

    // We just fetch the first network that's not public and vm-trafic for now, maybe we'll need to be
    // more selective later, let's see...
    for($i=0;$i<sizeof($machine_networks["networks"]);$i++){
        if($machine_networks["networks"][$i]["is_public"] == 'no' && $machine_networks["networks"][$i]["role"] != "vm-net" && $machine_networks["networks"][$i]["role"] != "ovs-bridge" && $machine_networks["networks"][$i]["role"] != "ceph-cluster" && $machine_networks["networks"][$i]["role"] != "ipmi" && $machine_networks["networks"][$i]["role"] != "vip"){
            $network_id = $machine_networks["networks"][$i]["id"];
            continue;
        }
    }

    // Get this machine's IP and hostname
    $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine  AND ips.network='$network_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "No such hostname in database when doing $q.";
        return $json;
    }
    $machine_netconf = mysqli_fetch_array($r);
    $machine_hostname = $machine_netconf["hostname"];
    $machine_ip = $machine_netconf["ipaddr"];

    // Get the API's VIP ip address
    $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND is_public='yes'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = "MySQL error when doing $q: " .mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find public network to find VIP when doing $q.";
        return $json;
    }

    $public_network = mysqli_fetch_array($r);
    $public_network_id = $public_network["id"];

    if($public_network["cidr"] == "32"){
        $vip_ipaddr  = $public_network["ip"];
        $vip_netmask = "32";
    }else{
        $q = "SELECT INET_NTOA(ips.ip) AS ipaddr FROM ips WHERE network='$public_network_id' AND usefor='vip' AND vip_usage='api'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "No such hostname in database when doing $q.";
            return $json;
        }
        $public_ip = mysqli_fetch_array($r);
        $vip_ipaddr  = $public_ip["ipaddr"];
        $vip_netmask = $public_network["cidr"];
    }
    if($cluster["vip_hostname"] == ""){
        $vip_hostname = $cluster["name"] . "-api." . $cluster["domain"];
    }else{
        $vip_hostname = $cluster["vip_hostname"];
    }

    $q = "SELECT * FROM networks WHERE id='$network_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = "MySQL error when doing $q: " .mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find network in database when doing $q.";
        return $json;
    }
    $network = mysqli_fetch_array($r);
    $network_ip = $network["ip"];
    $network_cidr = $network["cidr"];

    // Fetch all controllers

    $enc_amhn = "      all_masters:\n";
    $enc_amshn = "      all_masters_short_names:\n";
    $enc_amip = "      all_masters_ip:\n";
    $enc_nids = "      all_masters_ids:\n";
    $dns_all_masters = array();
    $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='controller' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' AND networks.role!='vip' ORDER BY hostname ASC";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $one_master = mysqli_fetch_array($r);
        $one_master_hostname = $one_master["hostname"];
        # Remove the cluster domain name from host hostname to find the short hostname.
        $one_master_short_hostname = str_replace("." . $cluster["domain"], "", $one_master_hostname);
        $one_master_ipaddr = $one_master["ipaddr"];
        $one_node_id = $one_master["nodeid"];
        $enc_amhn .= "         - $one_master_hostname\n";
        $enc_amshn .= "         - $one_master_short_hostname\n";
        $enc_amip .= "         - $one_master_ipaddr\n";
        $enc_nids .= "         - $one_node_id\n";
        $dns_all_masters[] = array( "host" => "$one_master_ipaddr", "port" => "5354", );
    }

    // Fetch all networks that may connect to the SQL nodes to write the firewall.
    $q_allowed_network = "SELECT DISTINCT networks.ip AS ip,networks.cidr AS cidr FROM networks,ips,machines WHERE ips.network = networks.id AND machines.id = ips.machine AND (machines.role='controller' OR machines.role='volume' OR machines.role='compute' OR machines.role='messaging') AND machines.cluster='$cluster_id' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' GROUP BY networks.ip ORDER BY length(networks.ip), networks.ip";
    $r_allowed_network = mysqli_query($con, $q_allowed_network);
    if($r_allowed_network === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_allowed_network = mysqli_num_rows($r_allowed_network);
    $enc_sql_allowed_networks = "      sql_allowed_networks:\n";
    for($i=0;$i<$n_allowed_network;$i++){
        $one_allowed_network = mysqli_fetch_array($r_allowed_network);
        $net = $one_allowed_network["ip"];
        $cidr = $one_allowed_network["cidr"];
        $enc_sql_allowed_networks .= "         - $net/$cidr\n";;
    }

    // Fetch all networks that contain an IP with a swiftstore or a swiftproxy
    $q_allowed_network = "SELECT DISTINCT networks.ip AS ip,networks.cidr AS cidr FROM networks,ips,machines WHERE ips.network = networks.id AND machines.id = ips.machine AND (machines.role='swiftstore' OR machines.role='swiftproxy') AND machines.cluster='$cluster_id'";
    $r_allowed_network = mysqli_query($con, $q_allowed_network);
    if($r_allowed_network === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_allowed_network = mysqli_num_rows($r_allowed_network);
    $enc_swift_storage_allowed_networks = "      swift_storage_allowed_networks:\n";
    for($i=0;$i<$n_allowed_network;$i++){
        $one_allowed_network = mysqli_fetch_array($r_allowed_network);
        $net = $one_allowed_network["ip"];
        $cidr = $one_allowed_network["cidr"];
        $enc_swift_storage_allowed_networks .= "         - $net/$cidr\n";;
    }

    // Fetch all networks that contain an IP that may access the API
    $q_no_api_rate_limit_networks = "SELECT DISTINCT networks.ip AS ip,networks.cidr AS cidr FROM networks,ips,machines WHERE ips.network = networks.id AND machines.id = ips.machine AND (machines.role='compute' OR machines.role='controller' OR machines.role='volume' OR machines.role='network' OR machines.role='messaging' OR machines.role='swiftproxy') AND machines.cluster='$cluster_id' AND networks.role!='ipmi' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='vip'";
    $r_no_api_rate_limit_networks = mysqli_query($con, $q_no_api_rate_limit_networks);
    if($r_no_api_rate_limit_networks === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_no_api_rate_limit_networks = mysqli_num_rows($r_no_api_rate_limit_networks);
    $enc_no_api_rate_limit_networks = "      no_api_rate_limit_networks:\n";
    for($i=0;$i<$n_no_api_rate_limit_networks;$i++){
        $one_no_api_rate_limit_network = mysqli_fetch_array($r_no_api_rate_limit_networks);
        $net = $one_no_api_rate_limit_network["ip"];
        $cidr = $one_no_api_rate_limit_network["cidr"];
        $enc_no_api_rate_limit_networks .= "         - $net/$cidr\n";
    }

    // Fetch all swift stores
    $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE ips.network=networks.id AND networks.role!='ipmi' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='vip' AND machines.role='swiftstore' AND machines.cluster='$cluster_id' AND machines.id=ips.machine";
    //        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips WHERE machines.role='swiftstore' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network='$network_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    $enc_allswiftstore_hostanme = "";
    $enc_allswiftstore_ip = "";
    for($i=0;$i<$n;$i++){
        $one_swiftstore = mysqli_fetch_array($r);
        $one_swiftstore_hostname = $one_swiftstore["hostname"];
        $one_swiftstore_ipaddr = $one_swiftstore["ipaddr"];
        $enc_allswiftstore_hostanme .= "         - $one_swiftstore_hostname\n";
        $enc_allswiftstore_ip .= "         - $one_swiftstore_ipaddr\n";
    }

    // Fetch all swift proxies
    $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE networks.id=ips.network AND networks.role!='ipmi' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND machines.role='swiftproxy' AND machines.cluster='$cluster_id' AND machines.id=ips.machine ORDER BY LENGTH(machines.hostname), machines.hostname";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    $enc_allswiftproxies_hostname = "";
    $enc_allswiftproxies_ip = "";
    for($i=0;$i<$n;$i++){
        $one_swiftproxy = mysqli_fetch_array($r);
        $one_swiftproxy_hostname = $one_swiftproxy["hostname"];
        $one_swiftproxy_ipaddr = $one_swiftproxy["ipaddr"];
        $one_node_id = $one_swiftproxy["nodeid"];
        $enc_allswiftproxies_hostname .= "         - $one_swiftproxy_hostname\n";
        $enc_allswiftproxies_ip .= "         - $one_swiftproxy_ipaddr\n";
    }

    // Get the number of compute nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='compute'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_compute_nodes = mysqli_num_rows($r);

    // Get the number of compute nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='sqlmsg'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_sqlmsg_nodes = mysqli_num_rows($r);

    // Get the number of metal nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='metal'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_metal_nodes = mysqli_num_rows($r);

    // Get the number of network nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='network'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_network_nodes = mysqli_num_rows($r);

    // Get the number of swiftstore nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='swiftstore'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_swiftstore_nodes = mysqli_num_rows($r);

    // Get the number of swiftproxy nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='swiftproxy'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_swiftproxy_nodes = mysqli_num_rows($r);

    // Get the number of volume nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='volume'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_volume_nodes = mysqli_num_rows($r);

    // Get the number of cephosd nodes
    $q = "SELECT * FROM machines WHERE (cluster='$cluster_id' AND role='cephosd') OR (cluster='$cluster_id' AND role='compute' AND compute_is_cephosd='yes')";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_cephosd_nodes = mysqli_num_rows($r);

    // If using Ceph AZ, get the list of Ceph AZ
    $ceph_az_list = Array();
    if($cluster["ceph_multiple_cluster_az"] == "yes"){
        $q = "SELECT UNIQUE(ceph_az) FROM machines WHERE cluster='$cluster_id' AND (role='cephosd' OR role='cephmon' OR (role='compute' AND compute_is_cephosd='yes'))";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $num_ceph_az = mysqli_num_rows($r);
        for($i=0;$i<$num_ceph_az;$i++){
            $a = mysqli_fetch_array($r);
            $ceph_az = $a["ceph_az"];
            $ceph_az_list[$ceph_az]["id"] = $ceph_az;

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$role='cephmon',$use_az='yes',$az=$ceph_az);
            if($ret["status"] != "success"){
                return $ret;
            }
            # This is the list of mon hosts and ips.
            $ceph_az_list[$ceph_az]["osd_members"] = $ret["data"]["osd_members"];
            $ceph_az_list[$ceph_az]["osd_ips"] = $ret["data"]["osd_ips"];
        }
    }

    // Get the number of cephmon nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='cephmon'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_cephmon_nodes = mysqli_num_rows($r);

    // Get the number of billosd nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='billosd'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_billosd_nodes = mysqli_num_rows($r);

    // Get the number of billosd nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='billmon'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_billmon_nodes = mysqli_num_rows($r);

    // Get the number of cephmon nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='cephmon'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_cephmon_nodes = mysqli_num_rows($r);

    // Get the number of radosgw nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='radosgw'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_radosgw_nodes = mysqli_num_rows($r);

    // Fetch all radosgw nodes
    $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE networks.id=ips.network AND networks.role!='ipmi' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND machines.role='radosgw' AND machines.cluster='$cluster_id' AND machines.id=ips.machine";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    $enc_allradosgw_hostname = "";
    $enc_allradosgw_ip = "";
    for($i=0;$i<$n;$i++){
        $one_radosgw = mysqli_fetch_array($r);
        $one_radosgw_hostname = $one_radosgw["hostname"];
        $one_radosgw_ipaddr = $one_radosgw["ipaddr"];
        $one_node_id = $one_radosgw["nodeid"];
        $enc_allradosgw_hostname .= "         - $one_radosgw_hostname\n";
        $enc_allradosgw_ip .= "         - $one_radosgw_ipaddr\n";
    }

    $enc_file = "---\n";
    $enc_file .= "classes:\n";

    ##############################################################
    ### Load hiera from /etc/openstack-cluster-installer/hiera ###
    ##############################################################
    $hiera_dir  = "/etc/openstack-cluster-installer/hiera";
    $hiera_role = $hiera_dir . "/roles/" . $machine_role . ".yaml";
    if(file_exists($hiera_role) === TRUE){
        $enc_file .= file_get_contents($hiera_role);
    }
    $hiera_node = $hiera_dir . "/nodes/" . $safe_hostname . ".yaml";
    if(file_exists($hiera_node) === TRUE){
        $enc_file .= file_get_contents($hiera_node);
    }
    $hiera_all  = $hiera_dir . "/all.yaml";
    if(file_exists($hiera_all) === TRUE){
        $enc_file .= file_get_contents($hiera_all);
    }
    $hiera_cluster_role = $hiera_dir . "/clusters/" . $cluster["name"] . "/roles/" . $machine_role . ".yaml";
    if(file_exists($hiera_cluster_role) === TRUE){
        $enc_file .= file_get_contents($hiera_cluster_role);
    }
    $hiera_cluster_node = $hiera_dir . "/clusters/" . $cluster["name"] . "/nodes/" . $safe_hostname . ".yaml";
    if(file_exists($hiera_cluster_node) === TRUE){
        $enc_file .= file_get_contents($hiera_cluster_node);
    }
    $hiera_cluster_all = $hiera_dir . "/clusters/" . $cluster["name"] . "/all.yaml";
    if(file_exists($hiera_cluster_all) === TRUE){
        $enc_file .= file_get_contents($hiera_cluster_all);
    }

    // Get the first_master controler node
    $first_master_machine_id = $cluster["first_master_machine_id"];
    $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='$first_master_machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find a single IP (found more or less than one IP) for this hostname's management network when doing: $q line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $first_master = mysqli_fetch_array($r);
    $first_master_hostname = $first_master["hostname"];
    $first_master_ipaddr   = $first_master["ipaddr"];

    // Do we have some messaging machines in the cluster
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='messaging'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_messaging_nodes = mysqli_num_rows($r);
    if($num_messaging_nodes > 0){
        $first_rabbit_machine_id = $cluster["first_rabbit_machine_id"];

        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='$first_rabbit_machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find first rabbit node when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $first_rabbit = mysqli_fetch_array($r);
        $first_rabbit_hostname = $first_rabbit["hostname"];
        $first_rabbit_ipaddr   = $first_rabbit["ipaddr"];

        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, machines.id AS nodeid FROM machines, ips, networks WHERE machines.role='messaging' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' ORDER BY hostname ASC";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)."  line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_rabbit_hostnames = "      all_rabbits:\n";
        $enc_rabbit_short_hostnames = "      all_rabbits_short_hostnames:\n";
        $enc_rabbit_ips = "      all_rabbits_ips:\n";
        $enc_rabbit_ids = "      all_rabbits_ids:\n";
        for($i=0;$i<$n;$i++){
            $one_rabbit = mysqli_fetch_array($r);
            $one_rabbit_hostname = $one_rabbit["hostname"];
            $enc_rabbit_hostnames .= "         - $one_rabbit_hostname\n";
            $sh = str_replace("." . $cluster["domain"], "", $one_rabbit_hostname);
            $enc_rabbit_short_hostnames .= "         - " . $sh . "\n";
            $one_rabbit_ipaddr = $one_rabbit["ipaddr"];
            $enc_rabbit_ips .= "         - $one_rabbit_ipaddr\n";
            $one_rabbit_id = $one_rabbit["nodeid"];
            $enc_rabbit_ids .= "         - $one_rabbit_id\n";
        }
        $enc_rabbits = $enc_rabbit_hostnames;
        $enc_rabbits .= $enc_rabbit_ips;

        // Fetch all other messaging (ie: all but this machine we're setting-up)
        $qnfmsg = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='messaging' AND machines.cluster='$cluster_id' AND machines.id != '".$machine["id"]."' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $rnfmsg = mysqli_query($con, $qnfmsg);
        if($rnfmsg === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $nnfmsg = mysqli_num_rows($rnfmsg);
        $enc_msg_other_nodes = "      messaging_other_nodes:\n";
        $enc_msg_other_nodes_ips = "      messaging_other_nodes_ips:\n";
        for($i=0;$i<$nnfmsg;$i++){
            $messaging_other_master = mysqli_fetch_array($rnfmsg);
            $messaging_other_master_hostname = $messaging_other_master["hostname"];
            $messaging_other_master_ipaddr = $messaging_other_master["ipaddr"];
            $enc_msg_other_nodes     .= "         - $messaging_other_master_hostname\n";
            $enc_msg_other_nodes_ips .= "         - $messaging_other_master_ipaddr\n";
        }
    }

    # Calculate the bridge list. If there's no network with bridge for this cluster, then we have a single br-ex.
    # Otherwise, we have a list of bridge from db.
    $q = "SELECT bridgename FROM networks WHERE cluster='$cluster_id' AND role='ovs-bridge'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $bridge_list = array();
    $n = mysqli_num_rows($r);
    if($n > 0){
        for($i=0;$i<$n;$i++){
            $one_network = mysqli_fetch_array($r);
            $bridge_list[] = $one_network["bridgename"];
        }
    }else{
        $bridge_list[] = "br-ex";
    }

    if($machine["role"] == "compute" || $machine["role"] == "network"){
        $enc_bridge_list = "      bridge_mapping_list:\n";
        $enc_external_netlist = "      external_network_list:\n";
        $neutron_external_network_name = $machine["neutron_external_network_name"];
        for($i=0;$i<sizeof($bridge_list);$i++){
            if($i == 0){
                $enc_bridge_list .= "         - $neutron_external_network_name:" . $bridge_list[$i] ."\n";
                $enc_external_netlist .= "         - $neutron_external_network_name\n";
            }else{
                $enc_bridge_list .= "         - $neutron_external_network_name$i:" . $bridge_list[$i] ."\n";
                $enc_external_netlist .= "         - $neutron_external_network_name$i\n";
            }
        }
        $enc_bridge_list .= $enc_external_netlist;
    }else{
        $enc_bridge_list = "      bridge_mapping_list:\n";
        $enc_external_netlist = "      external_network_list:\n";

        $q_ext_net_name = "SELECT DISTINCT neutron_external_network_name FROM machines WHERE (role='compute' OR role='network') AND cluster='$cluster_id'";
        $r_ext_net_name = mysqli_query($con, $q_ext_net_name);
        if($r_ext_net_name === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n_ext_net_name = mysqli_num_rows($r_ext_net_name);
        for($i_ext_net_name=0;$i_ext_net_name<$n_ext_net_name;$i_ext_net_name++){
            $a_ext_net_name = mysqli_fetch_array($r_ext_net_name);
            $neutron_external_network_name = $a_ext_net_name["neutron_external_network_name"];
            for($i=0;$i<sizeof($bridge_list);$i++){
                if($i == 0){
                    $enc_external_netlist .= "         - $neutron_external_network_name\n";
                    $enc_bridge_list .= "         - $neutron_external_network_name:" . $bridge_list[$i] ."\n";
                }else{
                    $enc_external_netlist .= "         - $neutron_external_network_name$i\n";
                    $enc_bridge_list .= "         - $neutron_external_network_name$i:" . $bridge_list[$i] ."\n";
                }
            }
        }
        $enc_bridge_list .= $enc_external_netlist;
    }

    $openstack_release = $conf["releasenames"]["openstack_release"];

    # See if we have machines with role SQL, if yes, find who's first_sql_machine_id.
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='sql'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_sql_machines = mysqli_num_rows($r);

    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='controller'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_controller_machines = mysqli_num_rows($r);

    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='sqlmsg'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_sqlmsg_machines = mysqli_num_rows($r);

    if($n_sqlmsg_machines > 0){
        // Get the first_sql sql node
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='" . $cluster["first_sqlmsg_machine_id"] . "' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find a single IP (found more or less than one IP) for this hostname's management network when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $first_sqlmsg = mysqli_fetch_array($r);
        $first_sqlmsg_hostname = $first_sqlmsg["hostname"];
        $first_sqlmsg_ipaddr   = $first_sqlmsg["ipaddr"];

        if($cluster["first_sqlmsg_machine_id"] == $machine_id){
            $is_first_sqlmsg = "true";
        }else{
            $is_first_sqlmsg = "false";
        }

        $enc_sqlmsg_ashn = "      all_sql:\n";
        $enc_sqlmsg_asip = "      all_sql_ip:\n";
        $enc_sqlmsg_snids = "      all_sql_ids:\n";

        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='sqlmsg' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_sqlmsg = mysqli_fetch_array($r);
            $one_sqlmsg_hostname = $one_sqlmsg["hostname"];
            $one_sqlmsg_ipaddr = $one_sqlmsg["ipaddr"];
            $one_sqlmsg_id = $one_sqlmsg["nodeid"];
            $enc_sqlmsg_ashn .= "         - $one_sqlmsg_hostname\n";
            $enc_sqlmsg_asip .= "         - $one_sqlmsg_ipaddr\n";
            $enc_sqlmsg_snids .= "         - $one_sqlmsg_id\n";
        }

        $enc_non_master_sqlmsg = "      non_master_sql:\n";
        $enc_non_master_sqlmsg_ip = "      non_master_sql_ip:\n";
        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='sqlmsg' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' AND machines.hostname!='$machine_hostname'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_sqlmsg = mysqli_fetch_array($r);
            $one_sqlmsg_hostname = $one_sqlmsg["hostname"];
            $one_sqlmsg_ipaddr = $one_sqlmsg["ipaddr"];
            $one_sqlmsg_id = $one_sqlmsg["nodeid"];
            $enc_non_master_sqlmsg .= "         - $one_sqlmsg_hostname\n";
            $enc_non_master_sqlmsg_ip .= "         - $one_sqlmsg_ipaddr\n";
        }

        $q = "SELECT INET_NTOA(ips.ip) AS ipaddr, networks.iface1 AS iface1, networks.iface2 AS iface2, networks.vlan AS vlan, networks.cidr AS cidr FROM ips,networks WHERE ips.usefor='vip' AND ips.vip_usage='sqlmsg' AND networks.cluster='$cluster_id' AND ips.network=networks.id";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find a single IP (found more or less than one IP) for this SQL's VIP when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $vip_sqlmsg = mysqli_fetch_array($r);
        $vip_sqlmsg_ip = $vip_sqlmsg["ipaddr"];
        $vip_sqlmsg_iface1 = $vip_sqlmsg["iface1"];
        $vip_sqlmsg_iface2 = $vip_sqlmsg["iface2"];
        $vip_sqlmsg_vlan   = $vip_sqlmsg["vlan"];
        $vip_sqlmsg_netmask= $vip_sqlmsg["cidr"];
        if(is_null($vip_sqlmsg_vlan) && $vip_sqlmsg_vlan != ""){
            $vip_sqlmsg_iface = "vlan" . $vip_sqlmsg_vlan;
        }else{
            if($vip_sqlmsg_iface2 != "none"){
                $vip_sqlmsg_iface = "bond0";
            }else{
                $vip_sqlmsg_iface = $vip_sqlmsg_iface1;
            }
        }
    }

    if($n_sql_machines > 0 || $n_controller_machines > 0){
        if($n_sql_machines > 0){
            $first_sql_machine_id = $cluster["first_sql_machine_id"];
            $vip_usage = "sql";
        }else{
            $first_sql_machine_id = $cluster["first_master_machine_id"];
            $vip_usage = "api";
        }
        if($first_sql_machine_id == $machine_id){
            $is_first_sql = "true";
        }else{
            $is_first_sql = "false";
        }

        // Get the first_sql sql node
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='$first_sql_machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find a single IP (found more or less than one IP) for this hostname's management network when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $first_sql = mysqli_fetch_array($r);
        $first_sql_hostname = $first_sql["hostname"];
        $first_sql_ipaddr   = $first_sql["ipaddr"];

        $enc_ashn = "      all_sql:\n";
        $enc_asip = "      all_sql_ip:\n";
        $enc_snids = "      all_sql_ids:\n";

        # If there's SQL machines, that's what we search for,
        # otherwise, we do the query on controllers.
        if($n_sql_machines > 0){
            $qrole = 'sql';
        }else{
            $qrole = 'controller';
        }
        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='$qrole' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_sql = mysqli_fetch_array($r);
            $one_sql_hostname = $one_sql["hostname"];
            $one_sql_ipaddr = $one_sql["ipaddr"];
            $one_sql_id = $one_sql["nodeid"];
            $enc_ashn .= "         - $one_sql_hostname\n";
            $enc_asip .= "         - $one_sql_ipaddr\n";
            $enc_snids .= "         - $one_sql_id\n";
        }

        $enc_non_master_sql = "      non_master_sql:\n";
        $enc_non_master_sql_ip = "      non_master_sql_ip:\n";
        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='$qrole' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' AND machines.hostname!='$machine_hostname'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_sql = mysqli_fetch_array($r);
            $one_sql_hostname = $one_sql["hostname"];
            $one_sql_ipaddr = $one_sql["ipaddr"];
            $one_sql_id = $one_sql["nodeid"];
            $enc_non_master_sql .= "         - $one_sql_hostname\n";
            $enc_non_master_sql_ip .= "         - $one_sql_ipaddr\n";
        }

        $q = "SELECT INET_NTOA(ips.ip) AS ipaddr, networks.iface1 AS iface1, networks.iface2 AS iface2, networks.vlan AS vlan, networks.cidr AS cidr FROM ips,networks WHERE ips.usefor='vip' AND ips.vip_usage='$vip_usage' AND networks.cluster='$cluster_id' AND ips.network=networks.id";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find a single IP (found more or less than one IP) for this SQL's VIP when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $vip_sql = mysqli_fetch_array($r);
        $vip_sql_ip = $vip_sql["ipaddr"];
        $vip_sql_iface1 = $vip_sql["iface1"];
        $vip_sql_iface2 = $vip_sql["iface2"];
        $vip_sql_vlan   = $vip_sql["vlan"];
        $vip_sql_netmask= $vip_sql["cidr"];
        if(is_null($vip_sql_vlan) && $vip_sql_vlan != ""){
            $vip_sql_iface = "vlan" . $vip_sql_vlan;
        }else{
            if($vip_sql_iface2 != "none"){
                $vip_sql_iface = "bond0";
            }else{
                $vip_sql_iface = $vip_sql_iface1;
            }
        }
    }elseif($n_controller_machines > 0){
        $first_sql_hostname = $first_master_hostname;
        $first_sql_ipaddr   = $first_master_ipaddr;
    }

    ###########################################
    ### Generic configuration for all nodes ###
    ###########################################
    $enc_file .= "   oci::generic:\n";
    $enc_file .= "      machine_hostname: $machine_hostname\n";
    $enc_file .= "      machine_role: ".$machine["role"]."\n";
    $enc_file .= "      openstack_release: $openstack_release\n";
    $enc_file .= "      admin_email_address: ".$cluster["admin_email_address"]."\n";
    $enc_file .= "      mail_relay_host: ".$cluster["mail_relay_host"]."\n";
    if ($cluster["mail_relay_enable"] == "yes") {
        $enc_file .= "      mail_relay_enable: true\n";
    }
#    $enc_file .= "      machine_ip: $machine_ip\n";

    if($conf["network"]["USE_HTTP_PROXY"]){
        $enc_file .= "      use_http_proxy: true\n";
    }else{
        $enc_file .= "      use_http_proxy: false\n";
    }
    $enc_file .= "      http_proxy_addr: " . $conf["network"]["HTTP_PROXY_ADDR"] . "\n";

    $enc_file .= "      ssh_listen_ips:\n";
    $enc_file .= "         - 127.0.0.1\n";
    $q = "SELECT INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='ovs-bridge' AND networks.role!='ipmi' AND networks.role!='ceph-cluster'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $one_ssh_listen_ip = mysqli_fetch_array($r);
        $ssh_listen_ip = $one_ssh_listen_ip["ipaddr"];
        $enc_file .= "         - $ssh_listen_ip\n";
    }

    $enc_file .= "      etc_hosts: ".base64_etc_hosts($con, $conf, $machine_id)."\n";
    $enc_file .= "      oci_repository_key: ".base64_encode(file_get_contents('/etc/openstack-cluster-installer/oci-repository-key.asc')) ."\n";
    $enc_file .= "      time_server_host: $cluster_time_server_host\n";

    # For compute hosts, we must ALWAYS generate an ssh an keypair for root,
    # because it's going to be used for libvirt live migration over TLS.
    if($cluster["generate_root_ssh_keys"] == "yes" || $machine["role"] == "compute"){
        $enc_file .= "      generate_root_ssh_keys: true\n";
        $json = get_ssh_keypair($con, $conf, $cluster_id, $machine_hostname, 'root');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_root_ssh_pub: " . unserialize($json["data"]["pubkey"]) . "\n";
        $enc_file .= "      pass_root_ssh_priv: " . base64_encode(unserialize($json["data"]["privatekey"])) . "\n";
    }else{
        $enc_file .= "      generate_root_ssh_keys: false\n";
    }

    # Give a list of interfaces, so that LLDP can be set to use only these
    $q = "SELECT name FROM ifnames WHERE machine_id='$machine_id'";
    $enc_file .= "      nics:\n";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $nic = mysqli_fetch_array($r);
        $nic_name = $nic["name"];
        $enc_file .= "         - $nic_name\n";
    }

    # If it's a compute host, we must gather all public ssh keys of the
    # compute hosts of the cluster, and transmit them through the ENC.
    # This way, all compute hosts can ssh all compute hosts, enabling
    # live migration over TLS with libvirt.
    if($machine["role"] == "compute"){
        $q = "SELECT hostname FROM machines WHERE cluster='$cluster_id' AND role='compute'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_file .= "      authorized_keys_hash:\n";
        for($i=0;$i<$n;$i++){
            $one_compute = mysqli_fetch_array($r);
            $one_compute_hostname = $one_compute["hostname"];
            $json = get_ssh_keypair($con, $conf, $cluster_id, $one_compute_hostname, 'root');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "         $one_compute_hostname: " . unserialize($json["data"]["pubkey"]) . "\n";
        }
        # If it's a compute, and we're in hyperconverged mode,
        # we must skip /etc/pki in the oci::generic setup if we're
        # entering oci::cephosd and not oci::compute. To do this,
        # we use this variable.
        if($num_cephosd_nodes > 0 && $num_cephmon_nodes == 0 && $cluster["initial_mon_on_ctrl"] == "yes" && $n != 0 && $machine["compute_is_cephosd"] == "yes"){
            $enc_file .= "      skip_etc_pki_setup: true\n";
        }
    }

    # Output the oci_facts.txt
    $ret = gen_oci_facts($con, $conf, $machine_id);
    if($ret["status"] != "success"){
        return $ret;
    }
    $oci_facts = $ret["data"];
    $enc_file .= "      oci_facts: " . base64_encode($oci_facts) . "\n";

    # Print out the authorized_keys_hash param
    if($machine["role"] == "volume" && $cluster["generate_root_ssh_keys"] == "yes"){
        $q = "SELECT hostname FROM machines WHERE role='volume' AND cluster='$cluster_id' AND id!='$machine_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_file .= "      authorized_keys_hash:\n";
        for($i=0;$i<$n;$i++){
            $one_volume = mysqli_fetch_array($r);
            $one_volume_hostname = $one_volume["hostname"];
            $json = get_ssh_keypair($con, $conf, $cluster_id, $one_volume_hostname, 'root');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "         $one_volume_hostname: " . unserialize($json["data"]["pubkey"]) . "\n";
        }
    }
    $key_path = "/var/lib/oci/ssl/slave-nodes/$machine_hostname/" . $machine_hostname . ".";
    if( (!file_exists($key_path . "key")) || (!file_exists($key_path . "crt")) || (!file_exists($key_path . "csr")) || (!file_exists($key_path . "pem")) ){
        if($machine_hostname == $vip_hostname){
            $WILDCARD = "--wildcard";
        }else{
            $WILDCARD = "";
        }
        $cmd = "sudo /usr/bin/oci-gen-slave-node-cert $WILDCARD $machine_hostname";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }

    # Ensure certificates are correctly generated if using SNI
    if ($cluster["haproxy_use_sni"] == "yes" && $machine_id == $first_master_machine_id){
        $cmd = "sudo /usr/bin/oci-check-cert-integrity $vip_hostname";
        $output = null;
        $return_var= 0;
        $expected_entries = [
            "DNS:$vip_hostname",
            "DNS:*.$vip_hostname",
        ];

        exec($cmd, $output, $return_val);

        $output = array_values(array_filter($output, function($line) {
            return !empty(trim($line));
        }));

        $sans_list = explode(",", str_replace(' ', '', $output[0]));

        $all_present = true;
        foreach ($expected_entries as $entry) {
            if (!in_array($entry, $sans_list)) {
                $all_present = false;
            }
        }

        if(!$all_present){
            $cmd = "sudo /usr/bin/oci-gen-slave-node-cert --wildcard $vip_hostname";
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);
        }
    }

    # Transmit the node itself key material (matching its hostname)
    $ssl_cert_snakeoil_key = file_get_contents("/var/lib/oci/ssl/slave-nodes/$machine_hostname/$machine_hostname.key");
    # What's bellow is *not* a mistake, it's really the .crt that goes into the .pem
    $ssl_cert_snakeoil_pem = file_get_contents("/var/lib/oci/ssl/slave-nodes/$machine_hostname/$machine_hostname.crt");
    $enc_file             .= "      ssl_cert_snakeoil_key: ".base64_encode($ssl_cert_snakeoil_key)."\n";
    $enc_file             .= "      ssl_cert_snakeoil_pem: ".base64_encode($ssl_cert_snakeoil_pem)."\n";

    # Check if a client certificate is missing. If it is, just generate it.
    $key_path = "/var/lib/oci/ssl/slave-nodes/$machine_hostname/" . $machine_hostname ."_client.";
    if((!file_exists($key_path . "key")) || (!file_exists($key_path . "crt")) || (!file_exists($key_path . "csr")) || (!file_exists($key_path . "pem"))){
        // Create the machine's SSL client cert
        $cmd = "sudo /usr/bin/oci-gen-slave-node-client-cert $machine_hostname";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }

    # Transmit the client certificate
    $ssl_client_cert_key = file_get_contents("/var/lib/oci/ssl/slave-nodes/$machine_hostname/" . $machine_hostname ."_client.key");
    $ssl_client_cert_crt = file_get_contents("/var/lib/oci/ssl/slave-nodes/$machine_hostname/" . $machine_hostname ."_client.crt");
    $ssl_client_cert_pem = file_get_contents("/var/lib/oci/ssl/slave-nodes/$machine_hostname/" . $machine_hostname ."_client.pem");

    $enc_file             .= "      ssl_cert_client_key: ".base64_encode($ssl_client_cert_key)."\n";
    $enc_file             .= "      ssl_cert_client_crt: ".base64_encode($ssl_client_cert_crt)."\n";
    $enc_file             .= "      ssl_cert_client_pem: ".base64_encode($ssl_client_cert_pem)."\n";

    # Transmit the OCI PKI to the cluster
    $oci_pki_oci_ca_chain  = file_get_contents('/var/lib/oci/ssl/ca/oci-pki-oci-ca-chain.pem');
    $oci_pki_oci_ca        = file_get_contents('/var/lib/oci/ssl/ca/oci-pki-oci-ca.pem');
    $oci_pki_root_ca       = file_get_contents('/var/lib/oci/ssl/ca/oci-pki-root-ca.pem');

    $enc_file             .= "      oci_pki_oci_ca_chain: ".base64_encode($oci_pki_oci_ca_chain)."\n";
    $enc_file             .= "      oci_pki_oci_ca: ".base64_encode($oci_pki_oci_ca)."\n";
    $enc_file             .= "      oci_pki_root_ca: ".base64_encode($oci_pki_root_ca)."\n";

    # Transmit the certs from the API
    $oci_pki_api_crt       = file_get_contents("/var/lib/oci/ssl/slave-nodes/$vip_hostname/$vip_hostname.crt");
    $oci_pki_api_csr       = file_get_contents("/var/lib/oci/ssl/slave-nodes/$vip_hostname/$vip_hostname.csr");
    $oci_pki_api_key       = file_get_contents("/var/lib/oci/ssl/slave-nodes/$vip_hostname/$vip_hostname.key");
    $oci_pki_api_pem       = file_get_contents("/var/lib/oci/ssl/slave-nodes/$vip_hostname/$vip_hostname.pem");

    $enc_file             .= "      oci_pki_api_crt: ".base64_encode($oci_pki_api_crt)."\n";
    $enc_file             .= "      oci_pki_api_csr: ".base64_encode($oci_pki_api_csr)."\n";
    $enc_file             .= "      oci_pki_api_key: ".base64_encode($oci_pki_api_key)."\n";
    $enc_file             .= "      oci_pki_api_pem: ".base64_encode($oci_pki_api_pem)."\n";

    # Transmit the certs for the swift proxies (which *may* be the same as the API
    # if no hostname has been set for it).
    if(strlen($cluster["swift_proxy_hostname"]) > 1){
        $swift_prx_hostname = $cluster["swift_proxy_hostname"];
    }else{
        $swift_prx_hostname = $vip_hostname;
    }
    $oci_pki_swiftproxy_key = file_get_contents("/var/lib/oci/ssl/slave-nodes/$swift_prx_hostname/$swift_prx_hostname.key");
    $oci_pki_swiftproxy_pem = file_get_contents("/var/lib/oci/ssl/slave-nodes/$swift_prx_hostname/$swift_prx_hostname.pem");

    $enc_file            .= "      oci_pki_swiftproxy_key: ".base64_encode($oci_pki_swiftproxy_key)."\n";
    $enc_file            .= "      oci_pki_swiftproxy_pem: ".base64_encode($oci_pki_swiftproxy_pem)."\n";

    $enc_file .= "\n";

    $q = "SELECT ip FROM networks WHERE cluster='$cluster_id' AND role='vip' AND vip_usage='controller'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find IP address of the controller VIP (found $n IPs), line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $ip_controller_vip_array = mysqli_fetch_array($r);
    $ip_controller_vip   = $ip_controller_vip_array["ip"];

    ########################
    ### Filebeat logging ###
    ########################
    $enc_file .= "   oci::filebeat:\n";
    $enc_file .= "      cluster_name: ".$cluster_name."\n";
    $enc_file .= "      machine_role: ".$machine["role"]."\n";
    $enc_file .= "      filebeat_repo: ".$conf["filebeat"]["filebeat_repo"]."\n";

    if($cluster["filebeat_enabled"] != "yes"){
        $enc_file .= "      filebeat_enable: false\n";
    }else{
        $enc_file .= "      filebeat_enable: true\n";
        # Define role and subroles
        # Note that logging roles may be different from actual roles,
        # for example in a controller it may have "swift" subrole, but
        # that's not relevant for logging.
        switch($machine["role"]){
        case "controller":
            if($num_compute_nodes > 0){
                $enc_file .= "      log_aodh: true\n";
                $enc_file .= "      log_apache2: true\n";
            }
            $enc_file .= "      log_barbican: true\n";
            $enc_file .= "      log_frr: true\n";
            if($num_cephosd_nodes > 0){
                $enc_file .= "      log_ceilometer: true\n";
                $enc_file .= "      log_ceph: true\n";
            }
            if($num_volume_nodes > 0 OR $num_cephosd_nodes > 0){
                $enc_file .= "      log_cinder: true\n";
            }
            if($num_cephosd_nodes > 0){
                $enc_file .= "      log_cloudkitty: true\n";
            }
            $enc_file .= "      log_corosync: true\n";
            if($cluster["install_magnum"] == "yes"){
                $enc_file .= "      log_etcd: true\n";
            }
            $enc_file .= "      log_haproxy: true\n";
            $enc_file .= "      log_horizon: true\n";
            if($cluster["install_designate"] == "yes"){
                $enc_file .= "      log_designate: true\n";
            }
            if($num_compute_nodes > 0){
                $enc_file .= "      log_glance: true\n";
            }
            if($num_cephosd_nodes > 0){
                $enc_file .= "      log_gnocchi: true\n";
            }
            $enc_file .= "      log_heat: true\n";
            $enc_file .= "      log_horizon: true\n";
            $enc_file .= "      log_keystone: true\n";
            $enc_file .= "      log_memcached: true\n";
            if($cluster["install_magnum"] == "yes"){
                $enc_file .= "      log_magnum: true\n";
            }
            if($n_sql_machines == 0){
                $enc_file .= "      log_mysql: true\n";
            }
            if($num_compute_nodes > 0){
                $enc_file .= "      log_neutron: true\n";
                $enc_file .= "      log_nova: true\n";
                $enc_file .= "      log_octavia: true\n";
            }
            if($num_network_nodes == 0 && $num_compute_nodes > 0){
                $enc_file .= "      log_openvswitch: true\n";
            }
            if($num_cephosd_nodes > 0){
                $enc_file .= "      log_panko: true\n";
            }
            $enc_file .= "      log_pacemaker: true\n";
            if($num_compute_nodes > 0){
                $enc_file .= "      log_placement: true\n";
            }
            $enc_file .= "      log_rabbitmq: true\n";
            if($num_cephosd_nodes > 0 || $cluster["install_designate"] == "yes"){
               $enc_file .= "      log_zookeeper: true\n";
            }
            break;
        case "messaging":
            $enc_file .= "      log_ceph: true\n";
            $enc_file .= "      log_cloudkitty: true\n";
            $enc_file .= "      log_haproxy: true\n";
            $enc_file .= "      log_gnocchi: true\n";
            $enc_file .= "      log_mysql: true\n";
            $enc_file .= "      log_rabbitmq: true\n";
            break;
        case "swiftproxy":
            $enc_file .= "      log_haproxy: true\n";
            $enc_file .= "      log_swiftproxy: true\n";
            if($machine["swift_store_account"] == "yes"){
                $enc_file .= "      log_swiftaccount: true\n";
            }
            if($machine["swift_store_container"] == "yes"){
                $enc_file .= "      log_swiftcontainer: true\n";
            }
            if($machine["swift_store_object"] == "yes"){
                $enc_file .= "      log_swiftobject: true\n";
            }
            break;
        case "swiftstore":
            if($machine["swift_store_account"] == "yes"){
                $enc_file .= "      log_swiftaccount: true\n";
            }
            if($machine["swift_store_container"] == "yes"){
                $enc_file .= "      log_swiftcontainer: true\n";
            }
            if($machine["swift_store_object"] == "yes"){
                $enc_file .= "      log_swiftobject: true\n";
            }
            break;
        case "network":
            $enc_file .= "      log_haproxy: true\n";
            $enc_file .= "      log_neutron: true\n";
            $enc_file .= "      log_openvswitch: true\n";
            break;
        case "compute":
            if($num_cephosd_nodes > 0){
                $enc_file .= "      log_cinder: true\n";
            }
            $enc_file .= "      log_haproxy: true\n";
            $enc_file .= "      log_neutron: true\n";
            $enc_file .= "      log_nova: true\n";
            $enc_file .= "      log_openvswitch: true\n";
            break;
        case "billosd":
        case "billmon":
        case "cephosd":
        case "cephmon":
            $enc_file .= "      log_ceph: true\n";
            break;
            break;
        case "radosgw":
            $enc_file .= "      log_ceph: true\n";
            break;
        case "volume":
            $enc_file .= "      log_cinder: true\n";
            break;
        case "debmirror":
        case "tempest":
        case "sql":
        case "sqlmsg":
        case "elastic":
            break;
        }

      $enc_file .= "      fields_under_root: ".$cluster['filebeat_fields_under_root']."\n";
#      $enc_file .= "      output_hosts: ".$cluster['filebeat_output_hosts']."\n";
      $enc_file .= "      output_hosts:\n";
      $qfb = "SELECT hostname,port FROM filebeathosts WHERE cluster='$cluster_id'";
      $rfb = $r = mysqli_query($con, $qfb);
      $nfb = mysqli_num_rows($rfb);
      for($i=0;$i<$nfb;$i++){
          $afb = mysqli_fetch_array($rfb);
          $fb_host = $afb["hostname"];
          $fb_port = $afb["port"];
          $enc_file .= "        - $fb_host:$fb_port\n";
      }
      $enc_file .= "      logging_level: ".$cluster['filebeat_logging_level']."\n";
      $enc_file .= "      logging_syslog: ".$cluster['filebeat_logging_syslog']."\n";
      $enc_file .= "      logging_files: ".$cluster['filebeat_logging_files']."\n";
      $enc_file .= "      logging_files_path: ".$cluster['filebeat_logging_files_path']."\n";
      $enc_file .= "      logging_files_name: ".$cluster['filebeat_logging_files_name']."\n";
    }
    $enc_file .= "\n";

    $skip_variables_dot_json_enc = "no";

    ###############################
    ### Role specific ENC output ##
    ###############################
    switch($machine["role"]){
    case "tempest":
        $enc_file .= "   oci::tempest:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";

        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        $enc_file .= "      glance_image_path: ".$conf["glance_image"]["image_path"]."\n";
        $enc_file .= "      glance_image_name: ".$conf["glance_image"]["image_name"]."\n";
        $enc_file .= "      glance_image_user: ".$conf["glance_image"]["image_ssh_user"]."\n";

        $enc_file .= "      neutron_external_network_name: ".$conf["tempest"]["external_network_name"]."\n";
        $enc_file .= "      neutron_floating_network_name: ".$conf["tempest"]["floating_network_name"]."\n";
        $enc_file .= "      neutron_public_network_name: ".$conf["tempest"]["public_network_name"]."\n";
        $enc_file .= "      nova_flavor_name: ".$conf["tempest"]["flavor_name"]."\n";
        $enc_file .= "      nova_flavor_alt_name: ".$conf["tempest"]["flavor_alt_name"]."\n";

        if($num_volume_nodes > 0 OR $num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_cinder: true\n";
        }

        if($num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_ceph: true\n";
        }

        if($num_compute_nodes > 0){
            $enc_file .= "      cluster_has_compute: true\n";
        }

        if($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
            $enc_file .= "      cluster_has_swift: true\n";
        }

        $json = get_cluster_password($con, $conf, $cluster_id, 'zabbix', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_zabbix_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

        $enc_file .= "      refstack_no_object_test_list: " . base64_encode(file_get_contents("/etc/oci-poc/platform.2021.11-test-list.txt")) . "\n";
        $enc_file .= "      refstack_object_test_list: " . base64_encode(file_get_contents("/etc/oci-poc/platform-object.2021.11-test-list.txt")) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rootrally', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_root_rally_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rally', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rally_db: " . $json["data"] . "\n";

        break;

    case "dns":
        $enc_file .= "   oci::dns:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";

        $enc_file .= "      vip_hostname: $vip_hostname\n";

        $enc_file .= "      vip_ipaddr: $ip_controller_vip\n";

        if($cluster["api_use_ipv6"] == "yes"){
            $enc_file .= "      api_use_ipv6: true\n";
            $enc_file .= "      vip6_ipaddr: ".$cluster["vip6_ipaddr"]."\n";
        }else{
            $enc_file .= "      api_use_ipv6: false\n";
        }

        # $all_masters and $all_masters_ip puppet variables:
        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'rndckey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_rndckey: " . $json["data"] . "\n";

        break;

    case "controller":
        if($machine_id == $first_master_machine_id){
            $is_first_master = "true";
        }else{
            $is_first_master = "false";
        }
        # If we're in hyperconverged mode, with cephmon on controllers
        # we may want to provision cephmon alone on the controllers first
        if ($num_cephosd_nodes > 0 && $num_cephmon_nodes == 0 && $cluster["initial_mon_on_ctrl"] == "yes"){
            $enc_file .= "   oci::cephmon:\n";
            $enc_file .= "      machine_hostname: $machine_hostname\n";
            $enc_file .= "      machine_ip: $machine_ip\n";
            $enc_file .= "      vip_hostname: $vip_hostname\n";
            $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
            $enc_file .= "      vip_netmask: $vip_netmask\n";
            $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
            $enc_file .= "      network_ipaddr: $network_ip\n";
            $enc_file .= "      network_cidr: $network_cidr\n";

            $servicetype = "ceph";
            $mon_node_type = 'cephmon';

            $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'fsid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'bootstraposdkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'adminkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'monkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'mgrkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mgr_key: " . $json["data"] . "\n";

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$mon_node_type);
            if($ret["status"] != "success"){
                return $ret;
            }
            $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
            $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";
            $skip_variables_dot_json_enc = "yes";
            break;
        }

        // Start writing oci::controller class parameters
        $enc_file .= "   oci::controller:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= $enc_bridge_list;
        if($num_compute_nodes > 0 && $num_network_nodes == 0){
            $enc_file .= "      machine_iface: br-ex\n";
        }else{
            $enc_file .= "      machine_iface: eth0\n";
        }

        $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";

        $enc_file .= "      first_sql: $first_sql_hostname\n";
        $enc_file .= "      first_sql_ip: $first_sql_ipaddr\n";
        $enc_file .= "      is_first_sql: $is_first_sql\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      sql_vip_netmask: $vip_sql_netmask\n";
        $enc_file .= "      sql_vip_iface: $vip_sql_iface\n";

        $enc_file .= "      initial_cluster_setup: $cluster_initial_cluster_setup\n";

        if($num_messaging_nodes > 0){
            $enc_file .= "      messaging_nodes_for_notifs: true\n";
            $enc_file .= $enc_rabbits;
        }
        $enc_file .= $enc_rabbit_short_hostnames;

        $enc_file .= $enc_ashn;
        $enc_file .= $enc_asip;
        $enc_file .= $enc_snids;
        $enc_file .= $enc_non_master_sql;
        $enc_file .= $enc_non_master_sql_ip;

        if($num_compute_nodes > 0 && $cluster["install_designate"] == "yes"){
            $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='dns' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' ORDER BY hostname ASC";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $n = mysqli_num_rows($r);
            $enc_dns_list = "      dns_nodes:\n";
            $enc_dnsip_list = "      dns_nodes_ip:\n";
            $nameserver_list = array();
            $dns_target_list = array();
            for($i=0;$i<$n;$i++){
                $one_dns = mysqli_fetch_array($r);
                $one_dns_hostname = $one_dns["hostname"];
                $one_dns_ip = $one_dns["ipaddr"];
                $enc_dns_list .= "         - $one_dns_hostname\n";
                $enc_dnsip_list .= "         - $one_dns_ip\n";
                $nameserver_list[] = array( "host" => $one_dns_ip, "port" => "53", );
                $dns_target_list[] = array( "type"        => "bind9",
                                            "description" => "BIND Instance for $one_dns_hostname",
                                            "masters"     => $dns_all_masters,
                                            "options"     => array( "host"          => "$one_dns_ip",
                                                                    "port"          => "53",
                                                                    "rndc_host"     => "$one_dns_ip",
                                                                    "rndc_port"     => "953",
                                                                    "rndc_key_file" => "/etc/designate/rndc.key",
                                                                  )
                                          );
            }
            $enc_file .= $enc_dns_list;
            $enc_file .= $enc_dnsip_list;

            $ns_records = array();
            $ns_records[] = array( "hostname" => $cluster["designate_ns1"].'.', "priority" => '100', );
            $ns_records[] = array( "hostname" => $cluster["designate_ns2"].'.', "priority" => '10', );

            $pools_dot_yaml = array(
                                array( "name" => "default",
                                       "description" => "OCI generated BIND Pool file",
                                       "nameservers" => $nameserver_list,
                                       "ns_records"  => $ns_records,
                                       "targets" => $dns_target_list,
                                     ),
                              );
            $enc_file .= "      designate_pools_yaml: " . base64_encode(yaml_emit($pools_dot_yaml)) ."\n";
        }


        $enc_file .= "      amp_secgroup_list: $amp_secgroup_list\n";
        $enc_file .= "      amp_boot_network_list: $amp_boot_network_list\n";

        if($cluster["cinder_backup_prefer_swift"] == "yes"){
            if($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                $enc_file .= "      cinder_backup_backend: swift\n";
            }elseif($num_cephosd_nodes > 0){
                $enc_file .= "      cinder_backup_backend: ceph\n";
            }else{
                $enc_file .= "      cinder_backup_backend: none\n";
            }
        }else{
            if($num_cephosd_nodes > 0){
                $enc_file .= "      cinder_backup_backend: ceph\n";
            }elseif($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                $enc_file .= "      cinder_backup_backend: swift\n";
            }else{
                $enc_file .= "      cinder_backup_backend: none\n";
            }
        }

        # No need for swift haproxy backend if no swift store.
        if($num_swiftstore_nodes == 0 && $num_swiftproxy_nodes == 0){
            $enc_file .= "      has_subrole_swift: false\n";
        }

        if($num_metal_nodes > 0){
            $enc_file .= "      has_subrole_ironic: true\n";
        }

        // Get all IPs from compute and volume nodes, that's needed to allow
        // SQL queries from them, as cinder-volume may run there.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='compute' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_comp_list = "      compute_nodes:\n";
        $enc_compip_list = "      compute_nodes_ip:\n";
        for($i=0;$i<$n;$i++){
            $one_compute = mysqli_fetch_array($r);
            $one_compute_hostname = $one_compute["hostname"];
            $one_compute_ip = $one_compute["ipaddr"];
            $enc_comp_list .= "         - $one_compute_hostname\n";
            $enc_compip_list .= "         - $one_compute_ip\n";
        }
        $enc_file .= $enc_comp_list;
        $enc_file .= $enc_compip_list;

        // Same with volume nodes
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='volume' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_volu_list = "      volume_nodes:\n";
        $enc_voluip_list = "      volume_nodes_ip:\n";
        for($i=0;$i<$n;$i++){
            $one_volume = mysqli_fetch_array($r);
            $enc_volu_hostname = $one_volume["hostname"];
            $enc_volu_ip = $one_volume["ipaddr"];
            $enc_volu_list .= "         - $enc_volu_hostname\n";
            $enc_voluip_list .= "         - $enc_volu_ip\n";
        }
        $enc_file .= $enc_volu_list;
        $enc_file .= $enc_voluip_list;

        $enc_file .= $enc_sql_allowed_networks;

        // Get the IP for VM trafic.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='vm-net' AND networks.role='vip'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 1){
            $vmnet_ip_array = mysqli_fetch_array($r);
            $vmnet_ip = $vmnet_ip_array["ipaddr"];
            $vmnet_mtu = $vmnet_ip_array["mtu"];
            $enc_file .= "      vmnet_ip: $vmnet_ip\n";
        // If we don't find a VMNet IP, then let's use the management network IP instead.
        }else{
            $enc_file .= "      vmnet_ip: $machine_ip\n";
        }

        $enc_file .= "      is_first_master: $is_first_master\n";
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";

        $enc_nfmc = "      non_first_master_controllers:\n";
        $enc_nfmc_ip = "      non_first_master_controllers_ip:\n";

        // Fetch all other controllers (ie: all but this machine we're setting-up)
        $qnfmc = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='controller' AND machines.cluster='$cluster_id' AND machines.id != '".$cluster["first_master_machine_id"]."' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $rnfmc = mysqli_query($con, $qnfmc);
        if($rnfmc === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $nnfmc = mysqli_num_rows($rnfmc);
        for($i=0;$i<$nnfmc;$i++){
            $one_other_master = mysqli_fetch_array($rnfmc);
            $one_other_master_hostname = $one_other_master["hostname"];
            $one_other_master_ipaddr = $one_other_master["ipaddr"];
            $enc_nfmc .= "         - $one_other_master_hostname\n";
            $enc_nfmc_ip .= "         - $one_other_master_ipaddr\n";
        }
        $enc_file .= $enc_nfmc;
        $enc_file .= $enc_nfmc_ip;

        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        if($cluster["swift_proxy_hostname"] == ""){
            $enc_file .= "      swiftproxy_hostname: none\n";
        }else{
            $enc_file .= "      swiftproxy_hostname: " . $cluster["swift_proxy_hostname"] ."\n";
        }
        $enc_file .= "      haproxy_custom_url: " . $cluster["haproxy_custom_url"] . "\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      other_masters:\n";

        $enc_omip  = "      other_masters_ip:\n";
        // Fetch all other controllers (ie: all but this machine we're setting-up)
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='controller' AND machines.cluster='$cluster_id' AND machines.id != '$machine_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_other_master = mysqli_fetch_array($r);
            $one_other_master_hostname = $one_other_master["hostname"];
            $one_other_master_ipaddr = $one_other_master["ipaddr"];
            $enc_file .= "         - $one_other_master_hostname\n";
            $enc_omip .= "         - $one_other_master_ipaddr\n";
        }
        $enc_file .= $enc_omip;

        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amshn;
        $enc_file .= $enc_amip;
        $enc_file .= $enc_nids;

        $enc_file .= "      all_swiftproxy:\n";
        $enc_file .= $enc_allswiftproxies_hostname;
        $enc_file .= "      all_swiftproxy_ip:\n";
        $enc_file .= $enc_allswiftproxies_ip;

        if($n_sql_machines > 0){
            $enc_file .= "      has_subrole_db: false\n";
        }

        if($num_compute_nodes > 0){
            $enc_file .= "      has_subrole_glance: true\n";
            $enc_file .= "      has_subrole_nova: true\n";
            $enc_file .= "      has_subrole_neutron: true\n";
            $enc_file .= "      has_subrole_aodh: true\n";
            $enc_file .= "      has_subrole_octavia: true\n";
            $enc_file .= "      has_subrole_magnum: true\n";
            $enc_file .= "      has_subrole_manila: true\n";
        }

        if($num_cephmon_nodes > 0){
            $enc_file .= "      cluster_has_mons: true\n";
        }else{
            $enc_file .= "      cluster_has_mons: false\n";
        }

        if($num_cephosd_nodes > 0 || $num_volume_nodes > 0){
            if($num_compute_nodes > 0){
                $enc_file .= "      has_subrole_cinder: true\n";
            }
            if($num_volume_nodes > 0){
                $enc_file .= "      cluster_has_cinder_volumes: true\n";
            }
            if ($cluster["ceph_use_cephadm_to_deploy"] == "no") {
                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'libvirtuuid');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_libvirtuuid: " . $json["data"] . "\n";
            }
        }
        if($num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_osds: true\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraposdkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'monkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'mgrkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mgr_key: " . $json["data"] . "\n";

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id,'cephmon');
            if($ret["status"] != "success"){
                return $ret;
            }
            $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
            $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";
            if($num_compute_nodes > 0){
                if($num_messaging_nodes > 0){
                    $enc_file .= "      has_subrole_gnocchi_services: false\n";
                }else{
                    $enc_file .= "      has_subrole_gnocchi_services: true\n";
                }
                $enc_file .= "      has_subrole_gnocchi: true\n";
                $enc_file .= "      has_subrole_ceilometer: true\n";
                $enc_file .= "      has_subrole_panko: true\n";
                $enc_file .= "      has_subrole_cloudkitty: true\n";
            }
            if($num_radosgw_nodes > 0){
                $enc_file .= "      has_subrole_radosgw: true\n";
                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraprgwkey');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_bootstrap_rgw_key: " . $json["data"] . "\n";
                $enc_file .= "      all_radosgw:\n";
                $enc_file .= $enc_allradosgw_hostname;
                $enc_file .= "      all_radosgw_ip:\n";
                $enc_file .= $enc_allradosgw_ip;
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'uuid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_gnocchi_rscuuid: " . $json["data"] . "\n";

        }else{
            $enc_file .= "      cluster_has_osds: false\n";
        }

        if($num_network_nodes == 0){
            $enc_file .= "      has_subrole_network_node: true\n";
        }else{
            $enc_file .= "      has_subrole_network_node: false\n";
        }

        if($cluster["glance_prefer_ceph_over_swift"] == "yes"){
            if($num_cephosd_nodes > 0){
                $glance_backend = 'ceph';
            }elseif($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                $glance_backend = 'swift';
            }else{
                $glance_backend = 'file';
            }
        }else{
            if($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                $glance_backend = 'swift';
            }elseif($num_cephosd_nodes > 0){
                $glance_backend = 'ceph';
            }else{
                $glance_backend = 'file';
            }
        }
        $enc_file .= "      glance_backend: " . $glance_backend . "\n";

        // Send all passwords
        $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'rootuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_rootuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'backup');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_backup: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'cookie');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_cookie: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_monitoring: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'credential1');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_credkey1: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'credential2');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_credkey2: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'fernetkey1');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_fernkey1: " . base64_encode(substr($json["data"],0,32)) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'fernetkey2');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_fernkey2: " . base64_encode(substr($json["data"],0,32)) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'ssh');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
        $enc_file .= "      pass_keystone_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'apidb');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_apidb: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'novaneutron', 'shared_secret');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_metadata_proxy_shared_secret: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_placement_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_placement_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'ssh');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
        $enc_file .= "      pass_glance_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'vrrpauth');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_vrrpauth: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'encryptkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_encryptkey: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'keystone_domain');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_keystone_domain: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'encryption');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_encryption: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'horizon', 'secretkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_horizon_secretkey: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'kek');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_kek: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mistral', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mistral_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mistral', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mistral_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mistral', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mistral_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'vmms', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_vmms_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'vmms', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_vmms_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'vmms', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_vmms_authtoken: " . $json["data"] . "\n";

        if ($cluster["ceph_use_cephadm_to_deploy"] == "no") {
            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";
        }

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'telemetry');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_telemetry: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'redis', 'redis');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_redis: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'heatbeatkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_heatbeatkey: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'manila', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_manila_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'manila', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_manila_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'manila', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_manila_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'domain');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_domain: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'rndckey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_rndckey: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ironic', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ironic_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ironic', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ironic_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ironic', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ironic_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ironicinspector', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ironic_inspector_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ironicinspector', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ironic_inspector_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ironicinspector', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ironic_inspector_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'zabbix', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_zabbix_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'radosgw', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_radosgw_authtoken: " . $json["data"] . "\n";

        $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";
        $enc_file .= "      enable_monitoring_graphs: " . $enable_monitoring_graphs . "\n";
        $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
        $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";

        $qgpu = "SELECT use_gpu,gpu_name,gpu_vendor_id,gpu_product_id,gpu_device_type FROM machines WHERE cluster='$cluster_id' AND use_gpu='yes'";
        $rgpu = mysqli_query($con, $qgpu);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $ngpu = mysqli_num_rows($rgpu);
        if($ngpu == 0){
            $enc_file .= "      use_gpu: false\n";
        }else{
            $enc_file .= "      use_gpu: true\n";
            $qgpu = "SELECT DISTINCT use_gpu,gpu_name,gpu_vendor_id,gpu_product_id,gpu_device_type FROM machines WHERE cluster='$cluster_id' AND use_gpu='yes'";
            $rgpu = mysqli_query($con, $qgpu);
            if($rgpu === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $ngpu = mysqli_num_rows($rgpu);
            $enc_gpu_def = "      gpu_def:\n";
            for($i=0;$i<$ngpu;$i++){
                $a_gpu = mysqli_fetch_array($rgpu);
                $gpu_names = explode(" ", $a_gpu["gpu_name"]);
                $gpu_vids = explode(" ", $a_gpu["gpu_vendor_id"]);
                $gpu_pids = explode(" ", $a_gpu["gpu_product_id"]);
                for ($j = 0; $j < count($gpu_names); $j++){
                    if (strpos($enc_gpu_def, $gpu_names[$j]) == false) {
                        $enc_gpu_def .= "         - " . $gpu_names[$j] . "." . $gpu_vids[$j] . "." . $gpu_pids[$j]  . "." . $a_gpu["gpu_device_type"] . "\n";
                    }
                }

            }
            $enc_file .= $enc_gpu_def;
        }

        if($cluster["extswift_use_external"] == "yes"){
            $enc_file .= "      extswift_use_external: true\n";
            $enc_file .= "      extswift_auth_url: " . $cluster["extswift_auth_url"] . "\n";
            $enc_file .= "      extswift_region: " . $cluster["extswift_region"] . "\n";
            $enc_file .= "      extswift_proxy_url: " . $cluster["extswift_proxy_url"] . "\n";
            $enc_file .= "      extswift_project_name: " . $cluster["extswift_project_name"] . "\n";
            $enc_file .= "      extswift_project_domain_name: " . $cluster["extswift_project_domain_name"] . "\n";
            $enc_file .= "      extswift_user_name: " . $cluster["extswift_user_name"] . "\n";
            $enc_file .= "      extswift_user_domain_name: " . $cluster["extswift_user_domain_name"] . "\n";
            $enc_file .= "      extswift_password: " . $cluster["extswift_password"] . "\n";
        }

        if($num_compute_nodes > 0 && $cluster["install_designate"] == "yes"){
            $enc_file .= "      has_subrole_designate: true\n";
        }

        # See if one node has Neutron dynamic routing, and install the extension in the
        # controller if we have one.
        $qdragent = "SELECT id FROM machines WHERE cluster='$cluster_id' AND neutron_install_dragent='yes'";
        $rdragent = mysqli_query($con, $qdragent);
        if($rdragent === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $ndragent = mysqli_num_rows($rdragent);
        if($ndragent == 0){
            $enc_file .= "      use_dynamic_routing: false\n";
        }else{
            $enc_file .= "      use_dynamic_routing: true\n";
        }

        # Transmit all the file names in /etc/openstack-cluster-installer/pollsters.d
        # so they can be included in the controller
        $pollsters_dir = '/etc/openstack-cluster-installer/pollsters.d';

        $dyn_pollster_files = "      dynamic_pollsters_files:\n";
        $dyn_pollster_names = "      dynamic_pollsters_names:\n";
        $dyn_pollster_data = "      dynamic_pollsters_data:\n";
        if ($dir_handle = opendir($pollsters_dir)) {
            while (false !== ($file_entry = readdir($dir_handle))) {
                if ($file_entry != "." && $file_entry != "..") {
                    $file_full_path = $pollsters_dir . "/" . $file_entry;
                    $file_content = file_get_contents($file_full_path);
                    $parsed_yaml = yaml_parse($file_content);
                    if($parsed_yaml !== FALSE){
                        $pollster_name = $parsed_yaml[0]["name"];
                        $dyn_pollster_files .= "        - $file_entry\n";
                        $dyn_pollster_names .= "        - $pollster_name\n";
                        $dyn_pollster_data .= "        - ". base64_encode($file_content) ."\n";
                    }
                }
            }
            closedir($dir_handle);
        }
        $enc_file .= $dyn_pollster_files;
        $enc_file .= $dyn_pollster_names;
        $enc_file .= $dyn_pollster_data;

        # Transmit a base64 version of /etc/ceilometer/gnocchi_resources.yaml
        # taken from /etc/openstack-cluster-installer/gnocchi_resources.yaml
        $gnocchi_resources_yaml_path = "/etc/openstack-cluster-installer/gnocchi_resources.yaml";
        if(file_exists($gnocchi_resources_yaml_path)){
            $gnocchi_resources_yaml_content = file_get_contents($gnocchi_resources_yaml_path);
            $enc_file .= "      gnocchi_resources_yaml: ".base64_encode($gnocchi_resources_yaml_content)."\n";
        }

        if($num_messaging_nodes >= 0){
            # Transmit a base64 version of /etc/cloudkitty/metrics.yml
            # taken from /etc/openstack-cluster-installer/metrics.yml
            $cloudkitty_metrics_yaml_path = "/etc/openstack-cluster-installer/metrics.yml";
            if(file_exists($cloudkitty_metrics_yaml_path)){
                $cloudkitty_metrics_yaml_content = file_get_contents($cloudkitty_metrics_yaml_path);
                $enc_file .= "      cloudkitty_metrics_yml: ".base64_encode($cloudkitty_metrics_yaml_content)."\n";
            }
        }
        $enc_file .= $enc_no_api_rate_limit_networks;

        $enc_file .= "      nameserver_v4_prim: ".$cluster["nameserver_v4_prim"]."\n"; 
        $enc_file .= "      nameserver_v6_prim: ".$cluster["nameserver_v6_prim"]."\n"; 
        $enc_file .= "      nameserver_v4_sec: ".$cluster["nameserver_v4_sec"]."\n"; 
        $enc_file .= "      nameserver_v6_sec: ".$cluster["nameserver_v6_sec"]."\n"; 
 
        break;

    case "messaging":
        if($machine_id == $first_rabbit_machine_id){
            $is_first_rabbit = "true";
        }else{
            $is_first_rabbit = "false";
        }

        // Start writing oci::controller class parameters
        $enc_file .= "   oci::messaging:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        # TODO: fix the correct iface name here.
        $enc_file .= "      machine_iface: eth0\n";

        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";

        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;

        $enc_file .= "      is_first_rabbit: $is_first_rabbit\n";
        $enc_file .= "      first_rabbit: $first_rabbit_hostname\n";
        $enc_file .= $enc_rabbits;
        $enc_file .= $enc_rabbit_ids;
        $enc_file .= $enc_rabbit_short_hostnames;

        $enc_file .= $enc_msg_other_nodes;
        $enc_file .= $enc_msg_other_nodes_ips;

        $enc_file .= "      initial_cluster_setup: $cluster_initial_cluster_setup\n";

        $ret = cluster_get_vip_ip($con, $conf, $cluster_id, 'messaging');
        if($ret["status"] != "success"){
            return $ret;
        }
        $enc_file .= "      messaging_vip_ipaddr: ".$ret["data"]["ipaddr"]."\n";
        $enc_file .= "      messaging_vip_network: ".$ret["data"]["network"]."\n";
        $enc_file .= "      messaging_vip_cidr: ".$ret["data"]["cidr"]."\n";

        if($num_sqlmsg_nodes > 0){
            $enc_file .= "      vip_sqlmsg_ip: $vip_sqlmsg_ip\n";
            $enc_file .= "      has_subrole_db: false\n";
        }else{
            $enc_file .= "      vip_sqlmsg_ip: ".$ret["data"]["ipaddr"]."\n";
            $enc_file .= "      has_subrole_db: true\n";
        }

        if($num_compute_nodes > 0){
            $enc_file .= "      has_subrole_glance: true\n";
            $enc_file .= "      has_subrole_nova: true\n";
            $enc_file .= "      has_subrole_neutron: true\n";
            $enc_file .= "      has_subrole_aodh: true\n";
            $enc_file .= "      has_subrole_octavia: true\n";
            $enc_file .= "      has_subrole_magnum: true\n";
            $enc_file .= "      has_subrole_manila: true\n";
        }

        if($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
            $enc_file .= "      has_subrole_swift: true\n";
        }

        if($num_cephosd_nodes > 0){
            $enc_file .= "      has_subrole_gnocchi: true\n";
            $enc_file .= "      has_subrole_ceilometer: true\n";
            $enc_file .= "      has_subrole_cloudkitty: true\n";
            $enc_file .= "      has_subrole_panko: true\n";
        }
        if($num_cephosd_nodes > 0 || $num_volume_nodes > 0){
            $enc_file .= "      has_subrole_cinder: true\n";
        }
        if($cluster["install_designate"] == "yes"){
            $enc_file .= "      has_subrole_designate: true\n";
        }

        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";

        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      sql_vip_netmask: $vip_sql_netmask\n";
        $enc_file .= "      sql_vip_iface: $vip_sql_iface\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'rootuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_rootuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'backup');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_backup: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'cookie');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_cookie: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_monitoring: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mistral', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mistral_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'vmms', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_vmms_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'manila', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_manila_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'uuid');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_rscuuid: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'zabbix', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_zabbix_authtoken: " . $json["data"] . "\n";

        # Ceph configuration
        if($num_billosd_nodes == 0){
            $servicetype = "ceph";
            $mon_node_type = 'cephmon';
        }else{
            $servicetype = "bill";
            $mon_node_type = 'billmon';
        }

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'fsid');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'bootstraposdkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'adminkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'openstackkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'monkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

        $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$mon_node_type);
        if($ret["status"] != "success"){
            return $ret;
        }
        $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
        $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

        # Transmit a base64 version of /etc/cloudkitty/metrics.yml
        # taken from /etc/openstack-cluster-installer/metrics.yml
        $cloudkitty_metrics_yaml_path = "/etc/openstack-cluster-installer/metrics.yml";
        if(file_exists($cloudkitty_metrics_yaml_path)){
            $cloudkitty_metrics_yaml_content = file_get_contents($cloudkitty_metrics_yaml_path);
            $enc_file .= "      cloudkitty_metrics_yml: ".base64_encode($cloudkitty_metrics_yaml_content)."\n";
        }

        $json = get_cluster_password($con, $conf, $cluster_id, 'elastickeystore', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_elastic_keystore: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_elastic: " . $json["data"] . "\n";

        break;

    case "sqlmsg":
        $enc_file .= "   oci::sql:\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      first_sql: $first_sqlmsg_hostname\n";
        $enc_file .= "      first_sql_ip: $first_sqlmsg_ipaddr\n";
        $enc_file .= "      is_first_sql: $is_first_sqlmsg\n";
        $enc_file .= "      sql_vip_ip: $vip_sqlmsg_ip\n";
        $enc_file .= "      sql_vip_netmask: $vip_sqlmsg_netmask\n";
        $enc_file .= "      sql_vip_iface: $vip_sqlmsg_iface\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        $q = "SELECT ip FROM networks WHERE cluster='$cluster_id' AND role='vip' AND vip_usage='messaging'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find IP address of the messaging VIP, line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $ip_messaging_vip_array = mysqli_fetch_array($r);
        $ip_messaging_vip   = $ip_messaging_vip_array["ip"];
        $enc_file .= "      ip_vip_to_whitelist: $ip_messaging_vip\n";

        $enc_file .= $enc_sql_allowed_networks;

        $enc_file .= $enc_sqlmsg_ashn;
        $enc_file .= $enc_sqlmsg_asip;
        $enc_file .= $enc_sqlmsg_snids;
        $enc_file .= $enc_non_master_sqlmsg;
        $enc_file .= $enc_non_master_sqlmsg_ip;

        $enc_file .= "      has_subrole_glance: false\n";
        $enc_file .= "      has_subrole_nova: false\n";
        $enc_file .= "      has_subrole_neutron: false\n";
        $enc_file .= "      has_subrole_aodh: false\n";
        $enc_file .= "      has_subrole_octavia: false\n";
        $enc_file .= "      has_subrole_cinder: false\n";
        $enc_file .= "      has_subrole_heat: false\n";
        $enc_file .= "      has_subrole_barbican: false\n";
        $enc_file .= "      has_subrole_mistral: false\n";
        $enc_file .= "      has_subrole_vmms: false\n";
        $enc_file .= "      has_subrole_designate: false\n";

        $enc_file .= "      has_subrole_gnocchi: true\n";
        $enc_file .= "      has_subrole_ceilometer: true\n";
        $enc_file .= "      has_subrole_cloudkitty: true\n";

        // Send all passwords
        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'rootuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_rootuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'backup');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_backup: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_db: " . $json["data"] . "\n";

        break;
    case "sql":
        $enc_file .= "   oci::sql:\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      first_sql: $first_sql_hostname\n";
        $enc_file .= "      first_sql_ip: $first_sql_ipaddr\n";
        $enc_file .= "      is_first_sql: $is_first_sql\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      sql_vip_netmask: $vip_sql_netmask\n";
        $enc_file .= "      sql_vip_iface: $vip_sql_iface\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        $enc_file .= "      ip_vip_to_whitelist: $ip_controller_vip\n";

        $enc_file .= $enc_sql_allowed_networks;

        $enc_file .= $enc_ashn;
        $enc_file .= $enc_asip;
        $enc_file .= $enc_snids;
        $enc_file .= $enc_non_master_sql;
        $enc_file .= $enc_non_master_sql_ip;

        if($num_compute_nodes > 0){
            $enc_file .= "      has_subrole_glance: true\n";
            $enc_file .= "      has_subrole_nova: true\n";
            $enc_file .= "      has_subrole_neutron: true\n";
            $enc_file .= "      has_subrole_aodh: true\n";
            $enc_file .= "      has_subrole_octavia: true\n";
        }else{
            $enc_file .= "      has_subrole_glance: false\n";
            $enc_file .= "      has_subrole_nova: false\n";
            $enc_file .= "      has_subrole_neutron: false\n";
            $enc_file .= "      has_subrole_aodh: false\n";
            $enc_file .= "      has_subrole_octavia: false\n";
        }
        if($num_cephosd_nodes > 0 || $num_volume_nodes > 0){
            $enc_file .= "      has_subrole_cinder: true\n";
        }else{
            $enc_file .= "      has_subrole_cinder: false\n";
        }
        if($num_cephosd_nodes > 0){
            $enc_file .= "      has_subrole_gnocchi: true\n";
            $enc_file .= "      has_subrole_ceilometer: true\n";
            $enc_file .= "      has_subrole_panko: true\n";
            $enc_file .= "      has_subrole_cloudkitty: true\n";
        }else{
            $enc_file .= "      has_subrole_gnocchi: false\n";
            $enc_file .= "      has_subrole_ceilometer: false\n";
            $enc_file .= "      has_subrole_panko: false\n";
            $enc_file .= "      has_subrole_cloudkitty: false\n";
        }
        $enc_file .= "      has_subrole_heat: true\n";
        $enc_file .= "      has_subrole_barbican: true\n";
        $enc_file .= "      has_subrole_mistral: true\n";
        $enc_file .= "      has_subrole_vmms: true\n";

        if($cluster["install_designate"] == "yes"){
            $enc_file .= "      has_subrole_designate: true\n";
        }

        // Send all passwords
        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'rootuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_rootuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'backup');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_backup: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'apidb');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_apidb: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_placement_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mistral', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mistral_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'vmms', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_vmms_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_db: " . $json["data"] . "\n";

        $enc_file .= "      initial_cluster_setup: $cluster_initial_cluster_setup\n";

        break;

    case "swiftproxy":
        $enc_file .= "   oci::swiftproxy:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      statsd_hostname: $cluster_statsd_hostname\n";
        $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
        $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";
        $enc_file .= $enc_no_api_rate_limit_networks;
        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;

        if($num_cephosd_nodes > 0){
            $enc_file .= "      enable_ceilometer_middleware: true\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_swift_messaging: " . $json["data"] . "\n";

            if($num_messaging_nodes > 0){
                $enc_file .= "      messaging_nodes_for_notifs: true\n";
                $enc_file .= $enc_rabbits;
            }

        }

        $enc_file .= "      all_swiftstore:\n";
        $enc_file .= $enc_allswiftstore_hostanme;
        $enc_file .= "      all_swiftstore_ip:\n";
        $enc_file .= $enc_allswiftstore_ip;
        $enc_file .= "      all_swiftproxy:\n";
        $enc_file .= $enc_allswiftproxies_hostname;
        $enc_file .= "      all_swiftproxy_ip:\n";
        $enc_file .= $enc_allswiftproxies_ip;
        if($cluster["swift_proxy_hostname"] == ""){
            $enc_file .= "      swiftproxy_hostname: none\n";
        }else{
            $enc_file .= "      swiftproxy_hostname: " . $cluster["swift_proxy_hostname"] ."\n";
        }

        $enc_file .= "      swift_disable_encryption: " . $cluster["swift_disable_encryption"] ."\n";
        $enc_file .= $enc_swift_storage_allowed_networks;

        if($cluster["swift_public_cloud_middlewares"] == "yes"){
            $enc_file .= "      swift_public_cloud_middlewares: true\n";
        }else{
            $enc_file .= "      swift_public_cloud_middlewares: false\n";
        }


        $q = "SELECT swiftregions.id AS region_id FROM machines,locations,swiftregions WHERE locations.swiftregion=swiftregions.name AND locations.id=machines.location_id AND machines.id='$machine_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find swiftregion ID";
            return $json;
        }
        $swiftregion = mysqli_fetch_array($r);
        $swiftregion_id = $swiftregion["region_id"];
        $enc_file .= "      swiftregion_id: $swiftregion_id\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathsuffix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathsuffix: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathprefix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathprefix: " . $json["data"] . "\n";

        $enc_file .= "      swift_encryption_key_id: \"" . $cluster["swift_encryption_key_id"] . "\"\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

        $qexpir = "SELECT hostname FROM machines WHERE cluster='$cluster_id' AND role='swiftproxy' ORDER BY length(hostname),hostname LIMIT 1";
        $rexpir = mysqli_query($con, $qexpir);
        if($rexpir === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($rexpir);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find expirer";
            return $json;
        }
        $expirer_machine = mysqli_fetch_array($rexpir);
        $expirer_hostname = $expirer_machine["hostname"];
        if($machine_hostname == $expirer_hostname){
            $enc_file .= "      object_expirer_enable: true" . "\n";
        }else{
            $enc_file .= "      object_expirer_enable: false" . "\n";
        }

        # What type of storage on the proxy?
        if($machine["swift_store_account"] == "yes"){
            $enc_file .= "      swift_store_account: true" . "\n";
        }else{
            $enc_file .= "      swift_store_account: false" . "\n";
        }
        if($machine["swift_store_container"] == "yes"){
            $enc_file .= "      swift_store_container: true" . "\n";
        }else{
            $enc_file .= "      swift_store_container: false" . "\n";
        }
        if($machine["swift_store_object"] == "yes"){
            $enc_file .= "      swift_store_object: true" . "\n";
        }else{
            $enc_file .= "      swift_store_object: false" . "\n";
        }

        if($machine["swift_store_account"] == "yes" || $machine["swift_store_container"] == "yes" || $machine["swift_store_object"] == "yes"){
            $enc_file .= "      swift_object_replicator_concurrency: " . $swift_object_replicator_concurrency . "\n";
            $enc_file .= "      swift_rsync_connection_limit: " . $swift_rsync_connection_limit . "\n";

            $enc_file .= "      block_devices:\n";
            foreach (get_non_system_block_devices($con, $machine) as $hdd_name) {
                $enc_file .= "         - $hdd_name\n";
            }
            if($machine["use_oci_sort_dev"] == "yes"){
                $enc_file .= "      use_oci_sort_dev: true\n";
            }else{
                $enc_file .= "      use_oci_sort_dev: false\n";
            }
        }

        break;

    case "swiftstore":
        $enc_file .= "   oci::swiftstore:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      zoneid: $machine_location\n";
        $enc_file .= "      block_devices:\n";
        foreach (get_non_system_block_devices($con, $machine) as $hdd_name) {
            $enc_file .= "         - $hdd_name\n";
        }

        if($machine["use_oci_sort_dev"] == "yes"){
            $enc_file .= "      use_oci_sort_dev: true\n";
        }else{
            $enc_file .= "      use_oci_sort_dev: false\n";
        }

        # What type of storage on the store?
        if($machine["swift_store_account"] == "yes"){
            $enc_file .= "      swift_store_account: true" . "\n";
        }else{
            $enc_file .= "      swift_store_account: false" . "\n";
        }
        if($machine["swift_store_container"] == "yes"){
            $enc_file .= "      swift_store_container: true" . "\n";
        }else{
            $enc_file .= "      swift_store_container: false" . "\n";
        }
        if($machine["swift_store_object"] == "yes"){
            $enc_file .= "      swift_store_object: true" . "\n";
        }else{
            $enc_file .= "      swift_store_object: false" . "\n";
        }


        $enc_file .= "      statsd_hostname: $cluster_statsd_hostname\n";
        $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
        $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";

        $enc_file .= "      swift_object_replicator_concurrency: " . $swift_object_replicator_concurrency . "\n";
        $enc_file .= "      swift_rsync_connection_limit: " . $swift_rsync_connection_limit . "\n";

        $enc_file .= "      all_swiftstore:\n";
        $enc_file .= $enc_allswiftstore_hostanme;
        $enc_file .= "      all_swiftstore_ip:\n";
        $enc_file .= $enc_allswiftstore_ip;
        $enc_file .= "      all_swiftproxy:\n";
        $enc_file .= $enc_allswiftproxies_hostname;
        $enc_file .= "      all_swiftproxy_ip:\n";
        $enc_file .= $enc_allswiftproxies_ip;
        $enc_file .= $enc_swift_storage_allowed_networks;

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathsuffix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathsuffix: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathprefix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathprefix: " . $json["data"] . "\n";
        break;

    case "network":
        $enc_file .= "   oci::network:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= $enc_bridge_list;
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
        $enc_file .= "      cluster_domain: $cluster_domain\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";

        if($num_messaging_nodes > 0){
            $enc_file .= "      messaging_nodes_for_notifs: true\n";
            $enc_file .= $enc_rabbits;
        }

        // Get the IP for VM trafic.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='vm-net' AND networks.role='vip'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 1){
            $vmnet_ip_array = mysqli_fetch_array($r);
            $vmnet_ip = $vmnet_ip_array["ipaddr"];
            $enc_file .= "      vmnet_ip: $vmnet_ip\n";
        // If we don't find a VMNet IP, then let's use the management network IP instead.
        }else{
            $enc_file .= "      vmnet_ip: $machine_ip\n";
        }

        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'vrrpauth');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_vrrpauth: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'novaneutron', 'shared_secret');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_metadata_proxy_shared_secret: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

        $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";

        if($num_compute_nodes > 0 && $cluster["install_designate"] == "yes"){
            $enc_file .= "      has_subrole_designate: true\n";
        }

        # See if one node has Neutron dynamic routing, and install the extension in the
        # compute if we have one.
        $qdragent = "SELECT id FROM machines WHERE cluster='$cluster_id' AND neutron_install_dragent='yes'";
        $rdragent = mysqli_query($con, $qdragent);
        if($rdragent === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $ndragent = mysqli_num_rows($rdragent);
        if($ndragent == 0){
            $enc_file .= "      use_dynamic_routing: false\n";
        }else{
            $enc_file .= "      use_dynamic_routing: true\n";
        }

        break;

    case "compute":
        # See if there's some controllers in the clusters that aren't full installed yet
        $q = "SELECT COUNT(id) AS numnotinstalled FROM machines WHERE role='controller' AND cluster='$cluster_id' AND (status!='installed' OR puppet_status!='success');";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Num of line is not one.";
            return $json;
        }
        $a = mysqli_fetch_array($r);
        $n = $a["numnotinstalled"];
        # If that's the case, and we're setting-up an hyper-converged node,
        # then we install it as cephosd first.
        if(($num_cephosd_nodes > 0 && $num_cephmon_nodes == 0 && $cluster["initial_mon_on_ctrl"] == "yes") || ($n != 0 && $machine["compute_is_cephosd"] == "yes")){
            # As we're artificially using another role, the
            # ENC variables from variables.json will be added
            # to the wrong class. So we do this workaround:
            $skip_variables_dot_json_enc = "yes";

            $enc_file .= "   oci::cephosd:\n";
            $enc_file .= "      machine_hostname: $machine_hostname\n";
            $enc_file .= "      machine_ip: $machine_ip\n";
            $enc_file .= "      vip_hostname: $vip_hostname\n";
            $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
            $enc_file .= "      vip_netmask: $vip_netmask\n";
            $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
            $enc_file .= "      network_ipaddr: $network_ip\n";
            $enc_file .= "      network_cidr: $network_cidr\n";
            $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";

            if($machine["ceph_osd_initial_setup"] == "yes"){
                    $ceph_osd_initial_setup = "true";
            }else{
                    $ceph_osd_initial_setup = "false";
            }
            $enc_file .= "      ceph_osd_initial_setup: $ceph_osd_initial_setup\n";

            $enc_file .= "      block_devices:\n";
            foreach (get_non_system_block_devices($con, $machine) as $hdd_name) {
                $enc_file .= "         - $hdd_name\n";
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraposdkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'monkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id,'cephmon');
            if($ret["status"] != "success"){
                return $ret;
            }
            $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
            $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

            // Get the IP for the Ceph network.
            $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu, networks.ip AS networkaddr, networks.cidr AS networkcidr FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='ceph-cluster'";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $n = mysqli_num_rows($r);
            if($n == 1){
                $cephnet_ip_array = mysqli_fetch_array($r);
                $cephnet_ip          = $cephnet_ip_array["ipaddr"];
                $cephnet_mtu         = $cephnet_ip_array["mtu"];
                $cephnet_networkaddr = $cephnet_ip_array["networkaddr"];
                $cephnet_networkcidr = $cephnet_ip_array["networkcidr"];
                $enc_file .= "      use_ceph_cluster_net: true\n";
                $enc_file .= "      cephnet_ip: $cephnet_ip\n";
                $enc_file .= "      cephnet_network_addr: $cephnet_networkaddr\n";
                $enc_file .= "      cephnet_network_cidr: $cephnet_networkcidr\n";
                $enc_file .= "      cephnet_mtu: $cephnet_mtu\n";
            // If we don't find a ceph-cluster IP, then let's use the management network IP instead.
            }else{
                $enc_file .= "      use_ceph_cluster_net: false\n";
                $enc_file .= "      cephnet_ip: $machine_ip\n";
                $enc_file .= "      cephnet_mtu: 0\n";
            }
        }else{
            $enc_file .= "   oci::compute:\n";
            $enc_file .= "      region_name: $cluster_region_name\n";
            $enc_file .= "      openstack_release: $openstack_release\n";
            $enc_file .= "      machine_hostname: $machine_hostname\n";
            $enc_file .= "      cluster_name: $cluster_name\n";
            $enc_file .= "      machine_ip: $machine_ip\n";
            $enc_file .= $enc_bridge_list;
            $enc_file .= "      first_master: $first_master_hostname\n";
            $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
            $enc_file .= "      cluster_domain: $cluster_domain\n";
            $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
            $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";
            $enc_file .= "      number_of_cpu_sockets: " . $machine["cpu_sockets"] . "\n";

            if($cluster["extswift_use_external"] == "yes"){
                $enc_file .= "      extswift_use_external: true\n";
                $enc_file .= "      extswift_auth_url: " . $cluster["extswift_auth_url"] . "\n";
                $enc_file .= "      extswift_region: " . $cluster["extswift_region"] . "\n";
                $enc_file .= "      extswift_proxy_url: " . $cluster["extswift_proxy_url"] . "\n";
                $enc_file .= "      extswift_project_name: " . $cluster["extswift_project_name"] . "\n";
                $enc_file .= "      extswift_project_domain_name: " . $cluster["extswift_project_domain_name"] . "\n";
                $enc_file .= "      extswift_user_name: " . $cluster["extswift_user_name"] . "\n";
                $enc_file .= "      extswift_user_domain_name: " . $cluster["extswift_user_domain_name"] . "\n";
                $enc_file .= "      extswift_password: " . $cluster["extswift_password"] . "\n";
            }

            if($num_messaging_nodes > 0){
                $enc_file .= "      messaging_nodes_for_notifs: true\n";
                $enc_file .= $enc_rabbits;
            }

            if($machine["nested_virt"] == "cluster_value"){
                if($cluster["nested_virt"] == "yes"){
                    $nested_virt = "true";
                }else{
                    $nested_virt = "false";
                }
            }elseif($machine["nested_virt"] == "yes"){
                    $nested_virt = "true";
            }else{
                    $nested_virt = "false";
            }
            $enc_file .= "      nested_virt: $nested_virt\n";

            // Get the IP for VM trafic.
            $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='vm-net' AND networks.role='vip'";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $n = mysqli_num_rows($r);
            if($n == 1){
                $vmnet_ip_array = mysqli_fetch_array($r);
                $vmnet_ip = $vmnet_ip_array["ipaddr"];
                $enc_file .= "      vmnet_ip: $vmnet_ip\n";
            // If we don't find a VMNet IP, then let's use the management network IP instead.
            }else{
                $enc_file .= "      vmnet_ip: $machine_ip\n";
            }

            $enc_file .= $enc_amhn;
            $enc_file .= $enc_amip;
            $enc_file .= "      vip_hostname: $vip_hostname\n";
            $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
            $enc_file .= "      vip_netmask: $vip_netmask\n";
            $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
            $enc_file .= "      network_ipaddr: $network_ip\n";
            $enc_file .= "      network_cidr: $network_cidr\n";

            if($machine["compute_is_cephosd"] == "yes"){
                $enc_file .= "      use_ceph_if_available: true\n";
            }else{
                if($machine["use_ceph_if_available"] == "yes"){
                    $enc_file .= "      use_ceph_if_available: ".$machine["use_ceph_if_available"]."\n";
                }else{
                    $enc_file .= "      use_ceph_if_available: false\n";
                }
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_cinder_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

            if($num_cephosd_nodes > 0){
                $enc_file .= "      cluster_has_osds: true\n";
                $enc_file .= "      has_subrole_ceilometer: true\n";

                $enc_file .= "      has_subrole_manila: true\n";

                if($cluster["ceph_multiple_cluster_az"] == "yes"){
                    $az = $machine["ceph_az"];
                    $enc_file .= "      ceph_az: $az\n";
                    $enc_file .= "      ceph_multiple_az_libvirtdata:\n";
                    foreach ($ceph_az_list as $az_array) {
                        $az_id = $az_array["id"];

                        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'libvirtuuid', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if($json["status"] != "success"){ return $json; }
                        $ceph_libvirtdata = $json["data"] . " ";

                        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if($json["status"] != "success"){ return $json; }
                        $ceph_libvirtdata .= $json["data"];

                        $enc_file .= "       - $ceph_libvirtdata\n";
                    }

                    $enc_file .= "      ceph_multiple_az_cinder_conf:\n";
                    foreach ($ceph_az_list as $az_array) {
                        $az_id = $az_array["id"];
                        $ceph_az_config = "";
                        $ceph_az_config .= "       - az: " . $az_id . "\n";

                        $ceph_az_name = "az-" . $az_id;
                        $ceph_az_config .= "         az_name: " . $ceph_az_name . "\n";

                        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'libvirtuuid', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if ($json["status"] != "success") { return $json; }
                        $ceph_az_config .= "         libvirtuuid: " . $json["data"] . "\n";

                        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if($json["status"] != "success"){ return $json; }
                        $ceph_az_config .= "         fsid: " . $json["data"] . "\n";

                        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if($json["status"] != "success"){ return $json; }
                        $ceph_az_config .= "         admin_key: " . $json["data"] . "\n";

                        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if($json["status"] != "success"){ return $json; }
                        $ceph_az_config .= "         openstack_key: " . $json["data"] . "\n";

                        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraposdkey', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if($json["status"] != "success"){ return $json; }
                        $ceph_az_config .= "         bootstrap_osd_key: " . $json["data"] . "\n";

                        $ret = enc_get_mon_nodes($con,$conf,$cluster_id,'cephmon', $cluster["ceph_multiple_cluster_az"], $az_id);
                        if($ret["status"] != "success"){ return $ret; }

                        $ceph_az_config .= "         mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
                        $ceph_az_config .= "         mon_host: " . $ret["data"]["osd_ips"] . "\n";

                        $enc_file .= "$ceph_az_config";
                    }
                }else{
                    $az = 0;
                }

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid', $cluster["ceph_multiple_cluster_az"], $az);
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'libvirtuuid', $cluster["ceph_multiple_cluster_az"], $az);
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_libvirtuuid: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraposdkey', $cluster["ceph_multiple_cluster_az"], $az);
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey', $cluster["ceph_multiple_cluster_az"], $az);
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey', $cluster["ceph_multiple_cluster_az"], $az);
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'monkey', $cluster["ceph_multiple_cluster_az"], $az);
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

                $ret = enc_get_mon_nodes($con,$conf,$cluster_id,'cephmon', $cluster["ceph_multiple_cluster_az"], $az);
                if($ret["status"] != "success"){
                    return $ret;
                }
                $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
                $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'telemetry');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      pass_ceilometer_telemetry: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'messaging');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      pass_ceilometer_messaging: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'authtoken');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      pass_ceilometer_authtoken: " . $json["data"] . "\n";

            }else{
                $enc_file .= "      cluster_has_osds: false\n";
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_nova_messaging: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_nova_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'ssh');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_nova_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
            $enc_file .= "      pass_nova_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_glance_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'novaneutron', 'shared_secret');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_metadata_proxy_shared_secret: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_neutron_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'vrrpauth');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_neutron_vrrpauth: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_placement_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'manila', 'db');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_manila_db: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'manila', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_manila_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'manila', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_manila_messaging: " . $json["data"] . "\n";

            $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";
            $enc_file .= "      enable_monitoring_graphs: " . $enable_monitoring_graphs . "\n";
            $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
            $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";

            if($machine["use_gpu"] == "yes"){
                $enc_file .= "      use_gpu: true\n";
                $enc_file .= "      gpu_name: [ " . str_replace(' ', ',', $machine["gpu_name"]) . " ] \n";
                $enc_file .= "      gpu_vendor_id: [ \"" . str_replace(' ', '","', $machine["gpu_vendor_id"]) . "\" ] \n";
                $enc_file .= "      gpu_product_id: [ \"" . str_replace(' ', '","', $machine["gpu_product_id"]) . "\" ] \n";
                $enc_file .= "      gpu_device_type: " . $machine["gpu_device_type"] . "\n";
                $enc_file .= "      vfio_ids: " . str_replace(' ', ',', $machine["vfio_ids"]) . "\n";
            }else{
                $enc_file .= "      use_gpu: false\n";
            }

            if($num_network_nodes == 0 || $machine["force_dhcp_agent"] == "yes"){
                $enc_file .= "      has_subrole_dhcp_agent: true\n";
            }else{
                $enc_file .= "      has_subrole_dhcp_agent: false\n";
            }

            # CPU mode & model(s)
            if($machine["cpu_mode"] == "cluster_value"){
                $cpu_mode = $cluster["cpu_mode"];
            }else{
                $cpu_mode = $machine["cpu_mode"];
            }
            if($machine["cpu_model"] == "cluster_value"){
                $cpu_model = $cluster["cpu_model"];
            }else{
                $cpu_model = $machine["cpu_model"];
            }
            if($machine["cpu_model_extra_flags"] == "cluster_value"){
                $cpu_model_extra_flags = $cluster["cpu_model_extra_flags"];
            }else{
                $cpu_model_extra_flags = $machine["cpu_model_extra_flags"];
            }
            if($machine["hw_machine_type"] == "cluster_value"){
                $hw_machine_type = $cluster["hw_machine_type"];
            }else{
                $hw_machine_type = $machine["hw_machine_type"];
            }
            $enc_file .= "      cpu_mode: $cpu_mode\n";
            $enc_file .= "      cpu_model: $cpu_model\n";
            if($cpu_model_extra_flags != "" && $cpu_model_extra_flags != "none"){
                $enc_file .= "      cpu_model_extra_flags: $cpu_model_extra_flags\n";
            }
            $enc_file .= "      hw_machine_type: $hw_machine_type\n";

            # nova_skip_*
            if($machine["nova_skip_cpu_compare_on_dest"] == "cluster_value"){
                $nova_skip_cpu_compare_on_dest = $cluster["nova_skip_cpu_compare_on_dest"];
            }else{
                $nova_skip_cpu_compare_on_dest = $machine["nova_skip_cpu_compare_on_dest"];
            }
            $enc_file .= "      nova_skip_cpu_compare_on_dest: $nova_skip_cpu_compare_on_dest\n";

            if($machine["nova_skip_cpu_compare_at_startup"] == "cluster_value"){
                $nova_skip_cpu_compare_at_startup = $cluster["nova_skip_cpu_compare_at_startup"];
            }else{
                $nova_skip_cpu_compare_at_startup = $machine["nova_skip_cpu_compare_at_startup"];
            }
            $enc_file .= "      nova_skip_cpu_compare_at_startup: $nova_skip_cpu_compare_at_startup\n";

            if($machine["nova_skip_hypervisor_version_check_on_lm"] == "cluster_value"){
                $nova_skip_hypervisor_version_check_on_lm = $cluster["nova_skip_hypervisor_version_check_on_lm"];
            }else{
                $nova_skip_hypervisor_version_check_on_lm = $machine["nova_skip_hypervisor_version_check_on_lm"];
            }
            $enc_file .= "      nova_skip_hypervisor_version_check_on_lm: $nova_skip_hypervisor_version_check_on_lm\n";

            # Cinder backend
            if($cluster["cinder_backup_prefer_swift"] == "yes"){
                if($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                    $enc_file .= "      backup_backend: swift\n";
                }elseif($num_cephosd_nodes > 0){
                    $enc_file .= "      backup_backend: ceph\n";
                }else{
                    $enc_file .= "      backup_backend: none\n";
                }
            }else{
                if($num_cephosd_nodes > 0){
                    $enc_file .= "      backup_backend: ceph\n";
                }elseif($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                    $enc_file .= "      backup_backend: swift\n";
                }else{
                    $enc_file .= "      backup_backend: none\n";
                }
            }

            if($cluster["install_designate"] == "yes"){
                $enc_file .= "      has_subrole_designate: true\n";
            }

            if($machine["compute_is_cephosd"] == "yes"){
                $enc_file .= "      block_devices:\n";
                foreach (get_non_system_block_devices($con, $machine) as $hdd_name) {
                    $enc_file .= "         - $hdd_name\n";
                }
            }

            # See if one node has Neutron dynamic routing, and install the extension in the
            # compute if we have one.
            $qdragent = "SELECT id FROM machines WHERE cluster='$cluster_id' AND neutron_install_dragent='yes'";
            $rdragent = mysqli_query($con, $qdragent);
            if($rdragent === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $ndragent = mysqli_num_rows($rdragent);
            if($ndragent == 0){
                $enc_file .= "      use_dynamic_routing: false\n";
            }else{
                $enc_file .= "      use_dynamic_routing: true\n";
            }
        }
        break;

    case "billosd":
    case "cephosd":
        $enc_file .= "   oci::cephosd:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";

        if($machine["ceph_osd_initial_setup"] == "yes"){
                $ceph_osd_initial_setup = "true";
        }else{
                $ceph_osd_initial_setup = "false";
        }
        $enc_file .= "      ceph_osd_initial_setup: $ceph_osd_initial_setup\n";

        $enc_file .= "      block_devices:\n";
        foreach (get_non_system_block_devices($con, $machine) as $hdd_name) {
            $enc_file .= "         - $hdd_name\n";
        }

        if($machine["role"] == "cephosd"){
            $servicetype = "ceph";
            $mon_node_type = 'cephmon';
        }else{
            $servicetype = "bill";
            $mon_node_type = 'billmon';
        }

        if($cluster["ceph_multiple_cluster_az"] == "yes"){
            $az = $machine["ceph_az"];
            $mc = "yes";
            $enc_file .= "      ceph_az: $az\n";
        }else{
            $az = 0;
            $mc = "no";
        }

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'ssh', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cephadm_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
        $enc_file .= "      pass_cephadm_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'fsid', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'bootstraposdkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'adminkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'openstackkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'monkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

        $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$mon_node_type, $cluster["ceph_multiple_cluster_az"], $az);
        if($ret["status"] != "success"){
            return $ret;
        }
        $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
        $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

        // Get the IP for the cluster network.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu, networks.ip AS networkaddr, networks.cidr AS networkcidr FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='ceph-cluster' AND networks.role='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 1){
            $cephnet_ip_array = mysqli_fetch_array($r);
            $cephnet_ip          = $cephnet_ip_array["ipaddr"];
            $cephnet_mtu         = $cephnet_ip_array["mtu"];
            $cephnet_networkaddr = $cephnet_ip_array["networkaddr"];
            $cephnet_networkcidr = $cephnet_ip_array["networkcidr"];
            $enc_file .= "      use_ceph_cluster_net: true\n";
            $enc_file .= "      cephnet_ip: $cephnet_ip\n";
            $enc_file .= "      cephnet_network_addr: $cephnet_networkaddr\n";
            $enc_file .= "      cephnet_network_cidr: $cephnet_networkcidr\n";
            $enc_file .= "      cephnet_mtu: $cephnet_mtu\n";
        // If we don't find a ceph-cluster IP, then let's use the management network IP instead.
        }else{
            $enc_file .= "      use_ceph_cluster_net: false\n";
            $enc_file .= "      cephnet_ip: $machine_ip\n";
            $enc_file .= "      cephnet_mtu: 0\n";
        }

        break;

    case "billmon":
    case "cephmon":
        $enc_file .= "   oci::cephmon:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";

        if($machine["role"] == "cephmon"){
            $servicetype = "ceph";
            $mon_node_type = 'cephmon';
        }else{
            $servicetype = "bill";
            $mon_node_type = 'billmon';
        }

        if($cluster["ceph_multiple_cluster_az"] == "yes"){
            $az = $machine["ceph_az"];
            $mc = "yes";
            $enc_file .= "      ceph_az: $az\n";
        }else{
            $az = 0;
            $mc = "no";
        }

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'ssh', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cephadm_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
        $enc_file .= "      pass_cephadm_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'fsid', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'bootstraposdkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'adminkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'openstackkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'monkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'mgrkey', $mc, $az);
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mgr_key: " . $json["data"] . "\n";

        $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$mon_node_type,$cluster["ceph_multiple_cluster_az"],$az);
        if($ret["status"] != "success"){
            return $ret;
        }
        $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
        $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";
        break;

    case "radosgw":
        $enc_file .= "   oci::radosgw:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";
        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraprgwkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_bootstrap_rgw_key: " . $json["data"] . "\n";
        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

        $ret = enc_get_mon_nodes($con,$conf,$cluster_id);
        if($ret["status"] != "success"){ return $ret; }
        $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
        $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'radosgw', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_radosgw_authtoken: " . $json["data"] . "\n";

        // Get the IP for the Ceph network.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu, networks.ip AS networkaddr, networks.cidr AS networkcidr FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='ceph-cluster'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 1){
            $cephnet_ip_array = mysqli_fetch_array($r);
            $cephnet_ip          = $cephnet_ip_array["ipaddr"];
            $cephnet_mtu         = $cephnet_ip_array["mtu"];
            $cephnet_networkaddr = $cephnet_ip_array["networkaddr"];
            $cephnet_networkcidr = $cephnet_ip_array["networkcidr"];
            $enc_file .= "      use_ceph_cluster_net: true\n";
            $enc_file .= "      cephnet_ip: $cephnet_ip\n";
            $enc_file .= "      cephnet_network_addr: $cephnet_networkaddr\n";
            $enc_file .= "      cephnet_network_cidr: $cephnet_networkcidr\n";
            $enc_file .= "      cephnet_mtu: $cephnet_mtu\n";
        // If we don't find a ceph-cluster IP, then let's use the management network IP instead.
        }else{
            $enc_file .= "      use_ceph_cluster_net: false\n";
            $enc_file .= "      cephnet_ip: $machine_ip\n";
            $enc_file .= "      cephnet_mtu: 0\n";
        }
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        // haproxy
        if($cluster["swift_proxy_hostname"] == ""){
            $enc_file .= "      swiftproxy_hostname: none\n";
        }else{
            $enc_file .= "      swiftproxy_hostname: " . $cluster["swift_proxy_hostname"] ."\n";
        }
        $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";
        $enc_file .= $enc_no_api_rate_limit_networks;
        $enc_file .= "      all_radosgw:\n";
        $enc_file .= $enc_allradosgw_hostname;
        $enc_file .= "      all_radosgw_ip:\n";
        $enc_file .= $enc_allradosgw_ip;
        break;

    case "volume":
        $enc_file .= "   oci::volume:\n";
        $enc_file .= "      region_name: $cluster_region_name\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";

        $enc_file .= "      block_devices:\n";

        if($num_cephosd_nodes > 0){
            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'libvirtuuid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_libvirtuuid: " . $json["data"] . "\n";
        }

        if($num_messaging_nodes > 0){
            $enc_file .= "      messaging_nodes_for_notifs: true\n";
            $enc_file .= $enc_rabbits;
        }

        foreach (get_non_system_block_devices($con, $machine) as $hdd_name) {
            $enc_file .= "         - $hdd_name\n";
        }

        $enc_file .= "      enabled_backend_list:\n";
        if($machine["cinder_enabled_backends"] != "no-override"){
            $backend_names_array = explode(":", $machine["cinder_enabled_backends"]);
            foreach($backend_names_array as &$backend){
                $enc_file .= "         - $backend\n";
            }
        }

        $volume_node_number = explode("-", explode(".", $machine_hostname)[0])[2];
        if($machine["cinder_enabled_backends"] != "no-override"){
            $enc_file .= "      vgname: " . $cluster_name . "vol" . $volume_node_number . "vg" ."\n";
        }else{
            $enc_file .= "      vgname: " . $cluster_name . "vol" . $volume_node_number . "vg0" ."\n";
        }

        if($cluster["cinder_backup_prefer_swift"] == "yes"){
            if($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                $enc_file .= "      backup_backend: swift\n";
            }elseif($num_cephosd_nodes > 0){
                $enc_file .= "      backup_backend: ceph\n";
            }else{
                $enc_file .= "      backup_backend: none\n";
            }
        }else{
            if($num_cephosd_nodes > 0){
                $enc_file .= "      backup_backend: ceph\n";
            }elseif($num_swiftstore_nodes > 0 || $num_swiftproxy_nodes > 0){
                $enc_file .= "      backup_backend: swift\n";
            }else{
                $enc_file .= "      backup_backend: none\n";
            }
        }

        # Ceph stuff
        if($num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_osds: true\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id);
            if($ret["status"] != "success"){
                return $ret;
            }
            $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
            $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";
        }else{
            $enc_file .= "      cluster_has_osds: false\n";
        }

        # Normal cinder password stuff
        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

        if($cluster["extswift_use_external"] == "yes"){
            $enc_file .= "      extswift_use_external: true\n";
            $enc_file .= "      extswift_auth_url: " . $cluster["extswift_auth_url"] . "\n";
            $enc_file .= "      extswift_region: " . $cluster["extswift_region"] . "\n";
            $enc_file .= "      extswift_proxy_url: " . $cluster["extswift_proxy_url"] . "\n";
            $enc_file .= "      extswift_project_name: " . $cluster["extswift_project_name"] . "\n";
            $enc_file .= "      extswift_project_domain_name: " . $cluster["extswift_project_domain_name"] . "\n";
            $enc_file .= "      extswift_user_name: " . $cluster["extswift_user_name"] . "\n";
            $enc_file .= "      extswift_user_domain_name: " . $cluster["extswift_user_domain_name"] . "\n";
            $enc_file .= "      extswift_password: " . $cluster["extswift_password"] . "\n";
        }
        $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";

        break;

    case "debmirror":
        $enc_file .= "   oci::debmirror:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        break;

    case "elastic":
        $enc_file .= "   oci::elastic:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        $enc_file .= "      elastic_repo_url: " . $conf["elastic"]["elastic_repo_url"] . "\n";
        $enc_file .= "      elastic_version: " . $conf["elastic"]["elastic_version"] . "\n";
        $enc_file .= "      elastic_key_url: " . $conf["elastic"]["elastic_key_url"] . "\n";
        $enc_file .= "      elastic_key_id: " . $conf["elastic"]["elastic_key_id"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'elastickeystore', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_elastic_keystore: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_elastic: " . $json["data"] . "\n";

        break;

    default:
        // If we don't know the role, then it's a custom one. Just output the
        // "standard" enc yaml file.
        break;
    }

    if($skip_variables_dot_json_enc == "no"){
        # Add to the role whatever's in the db matching the json file
        $json_file = file_get_contents("/usr/share/openstack-cluster-installer/variables.json");
        $dec_json = json_decode($json_file, TRUE);

        # Parse the per-machine attributes in variables.json
        $items_json_array = $dec_json["machines"];
        $a_keys = array_keys($items_json_array);
        $n = sizeof($a_keys);
        for($i=0;$i<$n;$i++){
            $col = $a_keys[$i];
            $attr = $items_json_array[$col];

            # First, see if the current custom variable includes the role of the machine
            $col_roles = $attr["roles"];
            $nroles = sizeof($col_roles);
            for($j=0;$j<$nroles;$j++){
                $one_role_name = $col_roles[$j];
                if($machine["role"] == $one_role_name || $one_role_name == "all"){
                    switch($attr["type"]){
                    case "int":
                    case "float":
                    case "string":
                    case "fqdn":
                        $custom_var_value = $machine[$col];
                        break;
                    case "ip_list":
                        if($machine[$col] == "none"){
                            $custom_var_value = "";
                        }else{
                            $custom_var_value = $machine[$col];
                        }
                        break;
                    case "boolean":
                        if($machine[$col] == "yes"){
                            $custom_var_value = "true";
                        }else{
                            $custom_var_value = "false";
                        }
                        break;
                    default:
                        die("Variable type not handled by OCI's ENC.\n");
                        break;
                    }
                    if($attr["type"] == 'string'){
                        $enc_file .= "      $col: \"$custom_var_value\"\n";
                    }else{
                        $enc_file .= "      $col: $custom_var_value\n";
                    }
                }
            }
        }

        # Parse the per-cluster attributes in variables.json
        $items_json_array = $dec_json["clusters"];
        $a_keys = array_keys($items_json_array);
        $n = sizeof($a_keys);
        for($i=0;$i<$n;$i++){
            $col = $a_keys[$i];
            $attr = $items_json_array[$col];

            # First, see if the current custom variable includes the role of the machine
            if(isset($attr["roles"])){
                $col_roles = $attr["roles"];
                $nroles = sizeof($col_roles);
            }else{
                $nroles = 0;
            }
            for($j=0;$j<$nroles;$j++){
                $one_role_name = $col_roles[$j];
                if($machine["role"] == $one_role_name || $one_role_name == "all"){
                    switch($attr["type"]){
                    case "int":
                    case "float":
                    case "string":
                    case "fqdn":
                        $custom_var_value = $cluster[$col];
                        break;
                    case "ip_list":
                        if($cluster[$col] == "none"){
                            $custom_var_value = "";
                        }else{
                            $custom_var_value = $cluster[$col];
                        }
                        break;
                    case "boolean":
                        if($cluster[$col] == "yes"){
                            $custom_var_value = "true";
                        }else{
                            $custom_var_value = "false";
                        }
                        break;
                    default:
                        die("Variable type not handled by OCI's ENC.\n");
                        break;
                    }
                    if($attr["type"] == 'string'){
                        $enc_file .= "      $col: \"$custom_var_value\"\n";
                    }else{
                        $enc_file .= "      $col: $custom_var_value\n";
                    }
                }
            }
        }
    }

    # Finally send the ENC data
    $json["data"] = $enc_file;
    return $json;
}

?>
