<?php

function mkdir_if_not_exists($dir, $perms = 0777){
    if (! file_exists($dir)) {
        mkdir($dir, $perms);
    }
}

function cluster_register_vip($con, $conf, $cluster_id, $role_name){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND role='vip' AND vip_usage='$role_name'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    # Check if the VIP is already registered for this cluster
    for($i=0;$i<$n;$i++){
        $network = mysqli_fetch_array($r);
        $network_id = $network["id"];
        $qn = "SELECT * FROM ips WHERE network='$network_id' AND usefor='vip' AND vip_usage='$role_name'";
        $rn = mysqli_query($con, $qn);
        if($rn === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
        $nn = mysqli_num_rows($rn);
        if($nn == 0){
            $json = reserve_ip_address($con, $conf, $network_id, 0, 'vip', $role_name);
            if($json["status"] == "success"){
                return $json;
            }else{
                continue;
            }
        }elseif($nn == 1){
            return $json;
        }else{
            $json["status"] = "error";
            $json["message"] = "More than one VIP in the db";
            return $json;
        }
    }
    $json["status"] = "error";
    $json["message"] = "Could not reserve VIP for cluster $cluster_id and role $role_name";
    return $json;
}

function cluster_get_vip_ip($con, $conf, $cluster_id, $role_name){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND role='vip'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    # Check if the VIP is already registered for this cluster
    for($i=0;$i<$n;$i++){
        $network = mysqli_fetch_array($r);
        $network_id = $network["id"];
        $qn = "SELECT INET_NTOA(ip) AS ipaddr FROM ips WHERE network='$network_id' AND usefor='vip' AND vip_usage='$role_name'";
        $rn = mysqli_query($con, $qn);
        if($rn === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
        $nn = mysqli_num_rows($rn);
        if($nn == 1){
            $ip = mysqli_fetch_array($rn);
            $json["data"]["ipaddr"]  = $ip["ipaddr"];
            $json["data"]["network"] = $network["ip"];
            $json["data"]["cidr"]    = $network["cidr"];
            return $json;
        }
    }
    $json["status"] = "error";
    $json["message"] = "Could not find VIP for cluster $cluster_id and role $role_name";
    return $json;
}

# Remove all traces of a machine in the OCI db
# (useful, for example, when a machine has hardware failure
# and is physically replaced)
function machine_destroy($con, $conf, $machine_id){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con). " doing $q";
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n == 1){
        $machine = mysqli_fetch_array($r);
        machine_forget_root_pass($con, $conf, $machine);
        machine_forget_ipmi_pass($con, $conf, $machine);
    }

    $q = "DELETE FROM blockdevices WHERE machine_id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con). " doing $q";
        return $json;
    }

    $q = "DELETE FROM physblockdevices WHERE machine_id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con). " doing $q";
        return $json;
    }

    $q = "DELETE FROM ifnames WHERE machine_id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con). " doing $q";
        return $json;
    }

    $q = "DELETE FROM ips WHERE machine='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con). " doing $q";
        return $json;
    }

    $q = "DELETE FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con). " doing $q";
        return $json;
    }

    return $json;
}

function add_node_to_cluster($con, $conf, $machine_id, $cluster_id, $role_name, $location_id, $safe_machine_fixed_ip="none"){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    # Fetch cluster name and domain
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find cluster in database.";
        return $json;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_name = $cluster["name"];
    $cluster_domain = $cluster["domain"];

    if($cluster["ceph_multiple_cluster_az"] == "yes" && ($role_name == 'cephmon' || $role_name == 'cephosd')){
        $q = "SELECT ceph_az FROM machines WHERE id='$machine_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot select machine with id $machine_id in ".__FILE__." line ".__LINE__;
        }
        $a = mysqli_fetch_array($r);
        $role_with_az = $role_name . $a["ceph_az"];
        if($role_name == 'cephmon'){
            $role_host_middle_name = 'ceph' . $a["ceph_az"] . 'mon';
        }else{
            $role_host_middle_name = 'ceph' . $a["ceph_az"] . 'osd';
        }
    }else{
        $role_with_az = $role_name;
        $role_host_middle_name = $role_name;
    }

    # Fetch role ID
    $q = "SELECT * FROM roles WHERE name='$role_name'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n == 0){
        $json["status"] = "error";
        $json["message"] = "Cannot find role in database.";
        return $json;
    }
    $a = mysqli_fetch_array($r);
    $role_id = $a["id"];

    if($cluster["ceph_multiple_cluster_az"] == "yes" && ($role_name == 'cephmon' || $role_name == 'cephosd')){
        # Query once
        $q = "SELECT * FROM roles WHERE name='$role_with_az'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
        $n = mysqli_num_rows($r);
        # If role with az doesn't exist, create it.
        if($n == 0){
            $q = "INSERT INTO roles (name) VALUES ('$role_with_az')";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con);
                return $json;
            }
        }

        # Query a 2nd time, this time we're sure the role exists.
        $q = "SELECT * FROM roles WHERE name='$role_with_az'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 0){
            $json["status"] = "error";
            $json["message"] = "Cannot find role in database.";
            return $json;
        }
        $a = mysqli_fetch_array($r);
        $role_id = $a["id"];
    }

    # Get the role count for this role,
    # if the record doesn't exist, create it,
    # otherwise, increment, and finally, calculate
    # the current role count for the node we're adding.
    $q = "SELECT * FROM rolecounts WHERE cluster='$cluster_id' AND role='$role_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n == 0){
        $role_count = 0;
        $q = "INSERT INTO rolecounts (cluster, role, count) VALUES ('$cluster_id', '$role_id', '1')";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
        $safe_role_count_id = mysqli_insert_id($con);
        $safe_role_count = "1";
    }else{
        $a = mysqli_fetch_array($r);
        $safe_role_count = $a["count"] + 1;
        $safe_role_count_id = $a["id"];

        $q = "UPDATE rolecounts SET count='$safe_role_count' WHERE id='$safe_role_count_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
    }

    # Check if there's no controller yet in the cluster.
    # If there's none, set this machine as the first_master,
    # and reserve a VIP (not in cae of controller). Do the
    # same if sql, sqlmsg or messaging.
    $set_first_master = "no";
    switch($role_name){
    case "controller":
        $set_first_master = "yes";
        $fm_col = "first_master_machine_id";
        break;
    case "messaging":
        $set_first_master = "yes";
        $fm_col = "first_rabbit_machine_id";
        break;
    case "sql":
        $set_first_master = "yes";
        $fm_col = "first_sql_machine_id";
        break;
    case "sqlmsg":
        $set_first_master = "yes";
        $fm_col = "first_sqlmsg_machine_id";
        break;
    default:
        $set_first_master = "no";
        break;
    }
    if($set_first_master == "yes"){
        # Get number of $role_name
        $q = "SELECT id FROM machines WHERE role='$role_name' AND cluster='$cluster_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
        $n = mysqli_num_rows($r);
        # If zero, we set the new node as first master.
        if($n == 0){
            $q = "UPDATE clusters SET $fm_col='$machine_id' WHERE id='$cluster_id'";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con);
                return $json;
            }
            if($role_name != "controller"){
                cluster_register_vip($con, $conf, $cluster_id, $role_name);
            }
        }
    }

    # Finally perform the UPDATE query for the machine so
    # that it joins the cluster
    $safe_hostname = $cluster_name . "-" . $role_host_middle_name . "-" . $safe_role_count . "." . $cluster_domain;
    $q = "UPDATE machines SET cluster='$cluster_id', role='$role_name', hostname='$safe_hostname', location_id='$location_id' WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    # and assign IPs to it
    $json = slave_assign_all_networks_ip_addresses($con, $conf, $machine_id, "machine", $location_id, $safe_machine_fixed_ip);

    // Create the machine's SSL cert
    $cmd = "sudo /usr/bin/oci-gen-slave-node-cert $safe_hostname";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    // Create the machine's SSL client cert
    $cmd = "sudo /usr/bin/oci-gen-slave-node-client-cert $safe_hostname";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    // Insert the machine into the DNS
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find machine in database when trying to call machine_create_host_dns().";
        return $json;
    }
    $machine = mysqli_fetch_array($r);
    machine_create_host_dns($con, $conf, $machine);

    return $json;
}

function insert_ssh_key($con, $conf, $cluster_id, $hostname, $username){
    # Generate the keypair
    $tmp_file = tempnam("/tmp", "nova-ssh-key-");
    unlink($tmp_file);

    $cmd = "ssh-keygen -t rsa -f $tmp_file -P ''";
    $output = "";
    $return_var = 0;
    exec($cmd, $output, $return_var);

    $private_key = file_get_contents($tmp_file);
    $public_key = file_get_contents($tmp_file . ".pub");
    strtok($public_key, " ");
    $public_key = strtok(" ");
    unlink($tmp_file);
    unlink($tmp_file . ".pub");

    # Store it
    $q = "INSERT INTO sshkeypairs (clusterid, hostname, username, pubkey, privatekey) VALUES ('$cluster_id', '$hostname', '$username', '" . serialize($public_key) . "', '" . serialize($private_key) . "')";
    $r = mysqli_query($con, $q);
    return;
}

function insert_cluster_pass($con, $conf, $cluster_id, $service, $passtype, $multiple_ceph_as="no", $az=0){
    if($passtype != "ssh" && ($service == "ceph" || ($service == "gnocchi" && $passtype == "uuid") || $service == "bill")){
        if($passtype == "fsid" || $passtype == "libvirtuuid" || $passtype == "uuid"){
            $hex = sprintf('%04x%04x-%04x-%04x-%04x-%04x%04x%04x',
                           mt_rand(0, 0xffff), mt_rand(0, 0xffff),
                           mt_rand(0, 0xffff),
                           mt_rand(0, 0x0fff) | 0x4000,
                           mt_rand(0, 0x3fff) | 0x8000,
                           mt_rand(0, 0xffff), mt_rand(0, 0xffff), mt_rand(0, 0xffff) );
        }else{
            # Use ceph-authtool to generate the Ceph keys.
            # Using openssl + base64 didn't work. If someone
            # finds a way, that would avoid ceph-common as depends for OCI.
            $cmd = "ceph-authtool --gen-print-key";
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);
            $hex = $output[0];
        }
    }elseif($service == "barbican" && $passtype == "kek"){
        $key = random_bytes(32);
        $random_kek = strtr(base64_encode($key), '+/', '-_');
        $q = "INSERT INTO passwords (cluster, service, passtype, pass) VALUES ('$cluster_id', '$service', '$passtype', '" . $random_kek . "')";
        $r = mysqli_query($con, $q);
        return;
    }elseif($service == "designate" && $passtype == "rndckey"){
        $hex = base64_encode(random_bytes(32));
        $hex2 = base64_encode(random_bytes(32));
        $rndc_key = base64_encode(hash_hmac('sha256', $hex, $hex2, true));
        $q = "INSERT INTO passwords (cluster, service, passtype, pass) VALUES ('$cluster_id', '$service', '$passtype', '" . $rndc_key . "')";
        $r = mysqli_query($con, $q);
        return;
    }elseif(($service == "nova" || $service == "glance" || $service == "keystone" || $service == "ceph" || $service == "bill") && $passtype == "ssh"){
        # Generate the keypair
        $tmp_file = tempnam("/tmp", "nova-ssh-key-");
        unlink($tmp_file);

        $cmd = "ssh-keygen -t rsa -f $tmp_file -P ''";
        $output = "";
        $return_var = 0;
        exec($cmd, $output, $return_var);

        $private_key = file_get_contents($tmp_file);
        $public_key = file_get_contents($tmp_file . ".pub");
        strtok($public_key, " ");
        $public_key = strtok(" ");
        unlink($tmp_file);
        unlink($tmp_file . ".pub");

        # Store it
        if($multiple_ceph_as == "yes"){
            $q = "INSERT INTO passwords (cluster, service, passtype, passtxt1, passtxt2, az) VALUES ('$cluster_id', '$service', '$passtype', '" . serialize($public_key) . "', '" . serialize($private_key) . "', '$az')";
        }else{
            $q = "INSERT INTO passwords (cluster, service, passtype, passtxt1, passtxt2) VALUES ('$cluster_id', '$service', '$passtype', '" . serialize($public_key) . "', '" . serialize($private_key) . "')";
        }
        $r = mysqli_query($con, $q);
        return;
    }elseif($service == "keystone" && ($passtype == "credential1" || $passtype == "credential2")){
        $hex = base64_encode(openssl_random_pseudo_bytes(32, $crypto_strong));
    }else{
        $bytes = openssl_random_pseudo_bytes(32, $crypto_strong);
        $hex   = bin2hex($bytes);
    }
    if($multiple_ceph_as == "yes"){
        $q = "INSERT INTO passwords (cluster, service, passtype, pass, az) VALUES ('$cluster_id', '$service', '$passtype', '$hex', '$az')";
    }else{
        $q = "INSERT INTO passwords (cluster, service, passtype, pass) VALUES ('$cluster_id', '$service', '$passtype', '$hex')";
    }
    $r = mysqli_query($con, $q);
}

function new_cluster($con, $conf, $cluster_name, $cluster_domain){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    // Check if cluster exists
    $r = mysqli_query($con, "SELECT * FROM clusters WHERE name='$cluster_name'");
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 0){
        $json["status"] = "error";
        $json["message"] = "Error: cluster name $cluster_name already exists.";
        return $json;
    }

    // Create the cluster
    $q = "INSERT INTO clusters (name, domain) VALUES ('$cluster_name', '$cluster_domain'); ";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $cluster_id = mysqli_insert_id($con);

    // Provision passwords for later
    insert_cluster_pass($con, $conf, $cluster_id, 'haproxy',  'stats');
    insert_cluster_pass($con, $conf, $cluster_id, 'mysql',    'rootuser');
    insert_cluster_pass($con, $conf, $cluster_id, 'mysql',    'backup');
    insert_cluster_pass($con, $conf, $cluster_id, 'rabbitmq', 'cookie');
    insert_cluster_pass($con, $conf, $cluster_id, 'rabbitmq', 'monitoring');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'adminuser');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'credential1');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'credential2');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'fernetkey1');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'fernetkey2');
    insert_cluster_pass($con, $conf, $cluster_id, 'keystone', 'ssh');
    insert_cluster_pass($con, $conf, $cluster_id, 'glance',   'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'glance',   'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'glance',   'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'glance',   'ssh');
    insert_cluster_pass($con, $conf, $cluster_id, 'nova',     'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'nova',     'apidb');
    insert_cluster_pass($con, $conf, $cluster_id, 'nova',     'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'nova',     'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'nova',     'ssh');
    insert_cluster_pass($con, $conf, $cluster_id, 'novaneutron', 'shared_secret');
    insert_cluster_pass($con, $conf, $cluster_id, 'placement','db');
    insert_cluster_pass($con, $conf, $cluster_id, 'placement','authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'cinder',   'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'cinder',   'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'cinder',   'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'neutron',  'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'neutron',  'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'neutron',  'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'neutron',  'vrrpauth');
    insert_cluster_pass($con, $conf, $cluster_id, 'heat',     'encryptkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'heat',     'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'heat',     'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'heat',     'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'heat',     'keystone_domain');
    insert_cluster_pass($con, $conf, $cluster_id, 'swift',    'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'swift',    'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'swift',    'hashpathsuffix');
    insert_cluster_pass($con, $conf, $cluster_id, 'swift',    'hashpathprefix');
    insert_cluster_pass($con, $conf, $cluster_id, 'swift',    'encryption');
    insert_cluster_pass($con, $conf, $cluster_id, 'horizon',  'secretkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'barbican', 'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'barbican', 'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'barbican', 'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'barbican', 'kek');
    insert_cluster_pass($con, $conf, $cluster_id, 'mistral',  'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'mistral',  'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'mistral',  'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'vmms',     'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'vmms',     'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'vmms',     'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceph',     'fsid');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceph',     'libvirtuuid');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceph',     'adminkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceph',     'openstackkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceph',     'monkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceph',     'mgrkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceph',     'bootstraposdkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'bill',     'fsid');
    insert_cluster_pass($con, $conf, $cluster_id, 'bill',     'libvirtuuid');
    insert_cluster_pass($con, $conf, $cluster_id, 'bill',     'adminkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'bill',     'openstackkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'bill',     'monkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'bill',     'mgrkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'bill',     'bootstraposdkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceilometer','db');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceilometer','messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceilometer','authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'ceilometer','telemetry');
    insert_cluster_pass($con, $conf, $cluster_id, 'gnocchi',  'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'gnocchi',  'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'gnocchi',  'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'panko'  ,  'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'panko',    'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'panko',    'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'gnocchi',  'uuid');
    insert_cluster_pass($con, $conf, $cluster_id, 'cloudkitty','db');
    insert_cluster_pass($con, $conf, $cluster_id, 'cloudkitty','messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'cloudkitty','authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'redis',    'redis');
    insert_cluster_pass($con, $conf, $cluster_id, 'aodh',     'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'aodh',     'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'aodh',     'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'octavia',  'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'octavia',  'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'octavia',  'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'octavia',  'heatbeatkey');
    insert_cluster_pass($con, $conf, $cluster_id, 'magnum',   'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'magnum',   'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'magnum',   'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'magnum',   'domain');
    insert_cluster_pass($con, $conf, $cluster_id, 'manila',   'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'manila',   'messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'manila',   'authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'designate','db');
    insert_cluster_pass($con, $conf, $cluster_id, 'designate','messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'designate','authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'designate','rndckey');
    insert_cluster_pass($con, $conf, $cluster_id, 'ironic','db');
    insert_cluster_pass($con, $conf, $cluster_id, 'ironic','messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'ironic','authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'ironicinspector','db');
    insert_cluster_pass($con, $conf, $cluster_id, 'ironicinspector','messaging');
    insert_cluster_pass($con, $conf, $cluster_id, 'ironicinspector','authtoken');
    insert_cluster_pass($con, $conf, $cluster_id, 'rootrally', 'db');
    insert_cluster_pass($con, $conf, $cluster_id, 'rally', 'db');
    # Elasticsearch passwords
    insert_cluster_pass($con, $conf, $cluster_id, 'elastickeystore', 'monitoring');
    insert_cluster_pass($con, $conf, $cluster_id, 'cloudkitty', 'monitoring');

    $dir = "/var/lib/oci/clusters/$cluster_name";
    mkdir_if_not_exists($dir, 0700);

    // Provision an API SSL certificate
    $api_hostname = $cluster_name . "-api." . $cluster_domain;
    $cmd = "sudo /usr/bin/oci-gen-slave-node-cert $api_hostname";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    ##########################################
    ### Create an ssh key for this cluster ###
    ##########################################
    $ssh_key_dir = "/var/lib/oci/clusters/$cluster_name/ssh";
    mkdir_if_not_exists($ssh_key_dir, 0700);
    $cmd = "ssh-keygen -P '' -f $ssh_key_dir/id_rsa";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    #############################################
    ### Create an ssh CA key for this cluster ###
    #############################################
    $ssh_key_dir = "/var/lib/oci/clusters/$cluster_name/ssh";
    $cmd = "ssh-keygen -P '' -f $ssh_key_dir/ca";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    ####################################################################
    ### Add the ssh CA to /etc/ssh/ssh_known_hosts in the OCI server ###
    ####################################################################
    $cmd = "sudo oci-add-ssh-ca-to-known-hosts $cluster_name";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    return $json;
}

function cluster_delete($con, $conf, $cluster_name){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    $q = "SELECT * FROM clusters WHERE name='$cluster_name'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Error: cluster name doesn't exist.";
        return $json;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_id = $cluster["id"];

    $q = "DELETE FROM passwords WHERE cluster='$cluster_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }

    $q = "DELETE FROM clusters WHERE name='$cluster_name'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }

    return $json;
}

// Fetch all networks where the machine has IP in.
function slave_fetch_networks($con, $conf, $machine_id){
    $out = array(
        "status"   => "success",
        "message"  => "Succesfully fetched networks.",
        "networks" => array(),
    );
    // Fetch the machine
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find machine: $machine_id.";
        return $out;
    }
    $machine = mysqli_fetch_array($r);
    $cluster_id = $machine["cluster"];
    $role = $machine["role"];
    $location_id = $machine["location_id"];

    // Fetch its network
    $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND role='$role' AND location_id='$location_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n >= 1){
        for($i=0;$i<$n;$i++){
            $out["networks"][] = mysqli_fetch_array($r);
        }
    }else{
        $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND role='norole' AND location_id='$location_id'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $out["networks"][] = mysqli_fetch_array($r);
        }
    }

    $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND role='all' AND location_id='$location_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $out["networks"][] = mysqli_fetch_array($r);
    }

    if($role == 'network' or $role == 'compute'){
        $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND (role='vm-net' OR role='ovs-bridge') AND location_id='$location_id'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $out["networks"][] = mysqli_fetch_array($r);
        }
    }

    if((in_array($role, ['cephosd', 'cephmon', 'radosgw'])) || (($role == 'compute') && ($machine['compute_is_cephosd'] == 'yes'))){
        $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND (role='ceph-cluster') AND location_id='$location_id'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $out["networks"][] = mysqli_fetch_array($r);
        }
    }

    // If there's no network node, then the controller needs the VM trafic network
    if($role == 'controller'){
        $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='network'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n == 0){
            $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND (role='vm-net' OR role='ovs-bridge') AND location_id='$location_id'";
            $r = mysqli_query($con, $q);
            $n = mysqli_num_rows($r);
            for($i=0;$i<$n;$i++){
                $out["networks"][] = mysqli_fetch_array($r);
            }
        }
    }

    return $out;
}

function get_ip_of_machine_on_network($con, $conf, $network_id, $machine_id){
    $out = array(
        "status"  => "success",
        "message" => "Succesfully found IP adresses for that machine on that network.",
    );

    // Fetch the network
    $q = "SELECT * FROM networks WHERE id='$network_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find network: $network_id.";
        return $out;
    }
    $network = mysqli_fetch_array($r);

    // Fetch the machine
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find machine: $machine_id.";
        return $out;
    }
    $machine = mysqli_fetch_array($r);

    $q = "SELECT INET_NTOA(ip) AS ipaddr FROM ips WHERE network='$network_id' AND machine='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find an IP address for this machine on that network: $q";
        return $out;
    }
    $ip = mysqli_fetch_array($r);
    $out["data"] = $ip["ipaddr"];
    return $out;
}

// Reserve an IP address in the "ips" table.
// usefor can be either machine or vip, if vip, then machine_id must be zero.
function reserve_ip_address($con, $conf, $network_id, $machine_id, $usefor, $vip_usage="api", $safe_machine_fixed_ip="none"){
    $out = array(
        "status"  => "success",
        "message" => "Succesfully reserved IP adresses.",
    );

    $q = "SELECT * FROM networks WHERE id='$network_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find network: $network_id.";
        return $out;
    }
    $network = mysqli_fetch_array($r);
    $network_ip   = $network["ip"];
    $network_name = $network["name"];
    $network_cidr = $network["cidr"];

    $network_first_ip_long = $network["first_ip"];
    $network_last_ip_long = $network["last_ip"];

    $number_of_ip = 2 ** (32 - $network_cidr);
    $network_ip_long = ip2long($network_ip);

    if($network_cidr == 32){
        $number_of_useable_ip = 1;
    }elseif($network_cidr == 31){
        $number_of_useable_ip = 2;
    }elseif($network_cidr == 30){
        $number_of_useable_ip = 2;
    }else{
        $number_of_useable_ip = $number_of_ip - 3;
    }

    # If we have this condition, this means we're dealing with a record from an old
    # version of the OCI db: in such case, we just fill the network first_ip and last_ip
    # with a valid, calculated value.
    if($network_first_ip_long == 0 || $network_last_ip_long == 0){
        // Calculate first and last IP of the network
        $network_first_ip_long = $network_ip_long + 2;
        $network_last_ip_long = $network_ip_long + $number_of_ip - 2;

        $q = "UPDATE networks SET first_ip='$network_first_ip_long', last_ip='$network_last_ip_long' WHERE id='$network_id'";
        $r = mysqli_query($con, $q);
    }
    // ******** START SEMAPHORE ********
    $key   = ftok(__FILE__,'m');
    $mysem = sem_get($key);
    sem_acquire($mysem);

    // Check if there's either no IP left, or no IP provisionned yet.
    $q = "SELECT id FROM ips WHERE network='$network_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n >= ($number_of_useable_ip)){
        $out["status"]  = "error";
        $out["message"] = "No IP available in the network $network_name, line: ".__LINE__." file: ".__file__;
        sem_release($mysem);
        return $out;
    }
    // We want a fixed IP in the range: just check if it's available and reserve it
    if($safe_machine_fixed_ip != "none"){
        $safe_machine_fixed_ip_long = ip2long($safe_machine_fixed_ip);
        if($safe_machine_fixed_ip_long < $network_first_ip_long || $safe_machine_fixed_ip_long > $network_last_ip_long){
            $out["status"]  = "error";
            $out["message"] = "Fixed IP address not in the network range.";
            return $out;
        }
        $q = "SELECT id FROM ips WHERE ip='$safe_machine_fixed_ip_long';";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $out["status"] = "error";
            $out["message"] = mysqli_error($con);
            return $out;
        }
        $n = mysqli_num_rows($r);
        if($n != 0){
            $out["status"] = "error";
            $out["message"] = "IP address already in use: cannot reserve.";
            return $out;
        }
        $q = "INSERT INTO ips (usefor,network,type,ip,machine,vip_usage) VALUES ('$usefor', '$network_id', '4', '".ip2long($safe_machine_fixed_ip)."', '$machine_id', '$vip_usage')";
    }else{
        if($n == 0){
            // Just use the first IP
            $q = "INSERT INTO ips (usefor,network,type,ip,machine,vip_usage) VALUES ('$usefor', '$network_id', '4', '$network_first_ip_long','$machine_id','$vip_usage')";
        }else{
            // See if the first IP of the range can be used
            $q = "SELECT ips.ip FROM ips WHERE ip='$network_first_ip_long' AND ips.network='$network_id'";
            $r = mysqli_query($con, $q);
            $n = mysqli_num_rows($r);

            if($n == 0){
                $nextIP = $network_first_ip_long;
            }else{
                // Get first IP address available in the range (find out what "hole" we have, or get the last IP +1).
                $q = "SELECT MIN(ips.ip + 1) AS nextIP FROM ips LEFT JOIN ips tf1 ON ips.ip + 1 = tf1.ip WHERE tf1.ip IS NULL AND ips.network='$network_id'";
                $r = mysqli_query($con, $q);
                $n = mysqli_num_rows($r);
                if($n != 1){
                    $out["status"]  = "error";
                    $out["message"] = "Could not reserve new IP, couldn't find an available ip in network with ID $network_id line:" . __LINE__ . " file: " . __FILE__;
                    return $out;
                }
                $a = mysqli_fetch_array($r);
                $nextIP = $a["nextIP"];
                // Do a basic check, just in case...
                if($nextIP < $network_first_ip_long || $nextIP > $network_last_ip_long){
                    $out["status"]  = "error";
                    $out["message"] = "Could not reserve new IP, nextIP $nextIP is lower than network_first_ip_long $network_first_ip_long or greater than network_last_ip_long $network_last_ip_long: couldn't find an available ip in network with ID $network_id line: " . __LINE__ . " file: " . __FILE__." with query: $q";
                }
            }
            $q = "INSERT INTO ips (usefor,network,machine,type,vip_usage,ip) VALUES ('$usefor', '$network_id', '$machine_id', '4', '$vip_usage', '$nextIP')";
        }
    }
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $out["status"]  = "error";
        $out["message"] = "Could not reserve new IP, query: $q, error: ".mysqli_error($con);
        sem_release($mysem);
        return $out;
    }
    sem_release($mysem);
    // ******** END SEMAPHORE ********
    return $out;
}

function reserve_ip_to_all_slaves_of_network($con, $conf, $cluster_id, $network_id, $role){
    $out = array(
        "status" => "success",
        "message" => "Successfully assigned IP for network id $network_id and cluster $cluster_id.",
    );

    $q = "SELECT role,location_id FROM networks WHERE id='$network_id'";
    $r = mysqli_query($con, $q);
    $a = mysqli_fetch_array($r);
    if($a["role"] == "ovs-bridge"){
        $out["message"] = "Not allocating IP on an ovs-bridge network.";
        return $out;
    }
    $location_id = $a["location_id"];

    if($role == "vm-net"){
        $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='network'";
        $r = mysqli_query($con, $q);
        $a = mysqli_fetch_array($r);
        $n = mysqli_num_rows($r);
        if($n > 0){
            $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND (role='network' OR role='compute') AND location_id='$location_id'";
        }else{
            $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND (role='controller' OR role='compute') AND location_id='$location_id'";
        }
    }elseif($role == "ceph-cluster"){
        $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='cephosd' AND location_id='$location_id'";
    }elseif($role == "ovs-bridge"){
        return $out;
    }else{
        if($role != "all"){
            $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='$role' AND location_id='$location_id'";
        }else{
            $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND location_id='$location_id'";
        }
    }
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $machine = mysqli_fetch_array($r);
        $machine_id = $machine["id"];
        $ret = reserve_ip_address($con, $conf, $network_id, $machine_id, "machine");
        if($ret["status"] != "success"){
            return $ret;
        }
    }
    return $out;
}

function slave_assign_all_networks_ip_addresses($con, $conf, $id, $usefor, $location_id, $safe_machine_fixed_ip="none"){
    $out = array(
        "status"  => "success",
        "message" => "Succesfully reserved IP adresses.",
    );
    if($usefor != "machine" && $usefor != "vip"){
        $out["status"]  = "error";
        $out["message"] = "Parameter usefor should be either machine or vip.";
    }

    if($usefor == "vip"){
        // If usefor is vip, then $id will in fact contain a network ID
        $network_id = $id;
    }else{
        $machine_id = $id;
    }

    // Fetch the machine
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find machine: $machine_id.";
        return $out;
    }
    $machine = mysqli_fetch_array($r);
    $cluster_id = $machine["cluster"];
    $role = $machine["role"];

    $machine_networks = slave_fetch_networks($con, $conf, $machine_id);

    for($i=0;$i<sizeof($machine_networks["networks"]);$i++){
        $network_id   = $machine_networks["networks"][$i]["id"];
        if($machine_networks["networks"][$i]["is_public"] == "no" && $machine_networks["networks"][$i]["role"] != "ovs-bridge"){
            $ret = reserve_ip_address($con, $conf, $network_id, $machine_id, "machine", "api", $safe_machine_fixed_ip);
        }
        if($ret["status"] == "error"){
            return $ret;
        }
    }
    return $out;
}

function slave_fetch_network_config($con, $conf, $machine_id){
    $out = array(
        "status"  => "success",
        "message" => "Succesfully fetched network config.",
    );
    $machine_networks = slave_fetch_networks($con, $conf, $machine_id);
    for($i=0;$i<sizeof($machine_networks["networks"]);$i++){
        $q = "SELECT INET_NTOA(ip) AS ipaddr FROM ips WHERE network='". $machine_networks["networks"][$i]["id"] ."' AND machine='$machine_id'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n != 1){
            continue;
        }
        $a = mysqli_fetch_array($r);
        $machine_networks["networks"][$i]["ipaddr"] = $a["ipaddr"];
        $network_ip_long = ip2long($machine_networks["networks"][$i]["ip"]);
        $network_gateway_long = $network_ip_long + 1;
        $machine_networks["networks"][$i]["gateway"] = long2ip($network_gateway_long);
    }
    return $machine_networks;
}

function get_ethname_from_network_config($con, $conf, $machine_id, $iface_in, $lldp_match='None'){
        if ($lldp_match != 'None'){
            $filter_cond = "AND switch_hostname LIKE '$lldp_match'";
        }else{
            $filter_cond = "";
        }

        $iface = "";
        $qeth = "";
        switch($iface_in){
        case "eth0":
            $iface = "eth0";
            break;
        case "eth1":
            $iface = "eth1";
            break;
        case "eth2":
            $iface = "eth2";
            break;
        case "eth3":
            $iface = "eth3";
            break;
        case "eth4":
            $iface = "eth4";
            break;
        case "eth5":
            $iface = "eth5";
            break;
        case "eno0":
            $iface = "eno0";
            break;
        case "eno1":
            $iface = "eno1";
            break;
        case "eno2":
            $iface = "eno2";
            break;
        case "eno3":
            $iface = "eno3";
            break;
        case "eno4":
            $iface = "eno4";
            break;
        case "eno5":
            $iface = "eno5";
            break;
        case "10m1":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='10' $filter_cond ORDER BY name LIMIT 1";
            break;
        case "10m2":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='10' $filter_cond ORDER BY name LIMIT 1,1";
            break;
        case "10m3":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='10' $filter_cond ORDER BY name LIMIT 2,1";
            break;
        case "10m4":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='10' $filter_cond ORDER BY name LIMIT 3,1";
            break;
        case "100m1":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='100' $filter_cond ORDER BY name LIMIT 1";
            break;
        case "100m2":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='100' $filter_cond ORDER BY name LIMIT 1,1";
            break;
        case "100m3":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='100' $filter_cond ORDER BY name LIMIT 2,1";
            break;
        case "100m4":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='100' $filter_cond ORDER BY name LIMIT 3,1";
            break;
        case "1g1":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='1000' $filter_cond ORDER BY name LIMIT 1";
            break;
        case "1g2":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='1000' $filter_cond ORDER BY name LIMIT 1,1";
            break;
        case "1g3":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='1000' $filter_cond ORDER BY name LIMIT 2,1";
            break;
        case "1g4":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed='1000' $filter_cond ORDER BY name LIMIT 3,1";
            break;
        case "10g1":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed>='10000' $filter_cond ORDER BY name LIMIT 1";
            break;
        case "10g2":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed>='10000' $filter_cond ORDER BY name LIMIT 1,1";
            break;
        case "10g3":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed>='10000' $filter_cond ORDER BY name LIMIT 2,1";
            break;
        case "10g4":
            $qeth = "SELECT name FROM ifnames WHERE machine_id='$machine_id' AND max_speed>='10000' $filter_cond ORDER BY name LIMIT 3,1";
            break;
        }
        if($qeth != ""){
            $reth = mysqli_query($con, $qeth);
            $neth = mysqli_num_rows($reth);
            if($neth != 1){
                $out["status"]  = "error";
                $out["message"] = "Cannot find matching ethernet device for $iface_in";
                return $out;
            }
            $aeth = mysqli_fetch_array($reth);
            $iface = $aeth["name"];
        }
        return $iface;
}

function slave_install_server_os_command($con, $conf, $machine_id){
    $out = array(
        "status"  => "success",
        "message" => "Succesfully reserved IPs and generated command line.",
        "cmd"     => "",
    );
    // Fetch the machine
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find machine: $machine_id.";
        return $out;
    }
    $machine = mysqli_fetch_array($r);
    $cluster_id = $machine["cluster"];
    $role = $machine["role"];

    if($machine["dest_blk"] == "none"){
        // Get its block device
        $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name LIKE '%a'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n != 1){
            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' LIMIT 1";
            $r = mysqli_query($con, $q);
            $n = mysqli_num_rows($r);
            if($n != 1){
                $out["status"]  = "error";
                $out["message"] = "Cannot find block device: $q<br>";
                return $out;
            }
        }
        $a2 = mysqli_fetch_array($r);
        $install_hdd_name = $a2["name"];
    }else{
        $install_hdd_name = $machine["dest_blk"];
    }

    $machine_networks = slave_fetch_network_config($con, $conf, $machine_id);
    if(sizeof($machine_networks["networks"]) == 0){
        $out["status"]  = "error";
        $out["message"] = "No network configured for this machine.";
        return $out;
    }

    $has_vm_net = "no";
    $has_ovs_bridge = "no";
    $ovs_bridge_list = array();
    $has_cephnet = "no";
    for($i=0;$i<sizeof($machine_networks["networks"]);$i++){
        if($machine_networks["networks"][$i]["role"] == "vm-net"){
            $has_vm_net = "yes";
            if(isset($vm_net)){
                $out["status"]  = "error";
                $out["message"] = "Only a single VM trafic network can be set.";
                return $out;
            }
            $vm_net = $machine_networks["networks"][$i];
        }elseif($machine_networks["networks"][$i]["role"] == "ovs-bridge"){
            $has_ovs_bridge = "yes";
            $ovs_bridge_list[] = $machine_networks["networks"][$i];
        }elseif($machine_networks["networks"][$i]["role"] == "ceph-cluster"){
            if($has_cephnet == "yes"){
                $out["status"]  = "error";
                $out["message"] = "Only a single CephNET network can be set.";
                return $out;
            }
            $has_cephnet = "yes";
            $cephnet = $machine_networks["networks"][$i];
        }elseif($machine_networks["networks"][$i]["is_public"] == "yes"){
            continue;
        }else{
            if(isset($mgmt_net)){
                $out["status"]  = "error";
                $out["message"] = "Only a single management network can be set.";
                return $out;
            }
            $mgmt_net = $machine_networks["networks"][$i];
        }
    }

    $qc = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $rc = mysqli_query($con, $qc);
    $nc = mysqli_num_rows($rc);
    if($nc != 1){
        $out["status"]  = "error";
        $out["message"] = "Cannot find cluster: $cluster_id.";
        return $out;
    }
    $cluster = mysqli_fetch_array($rc);

    if(($cluster["bgp_to_the_host"] == "yes" && $machine["force_no_bgp2host"] == "no") || $machine["force_frr"] == "yes"){
        $asnum = ",asnum=" . $cluster["asn"];
    }else{
        $asnum = "";
    }


    $network_params = "";
    if($has_vm_net == "yes"){
        if(!isset($mgmt_net)){
            $out["status"]  = "error";
            $out["message"] = "No management network is set.";
            return $out;
        }
        $br_ex_ovs_bridged_network = "yes";

        if($cluster["use_ovs_ifaces"] == "yes"){
            $use_ovs_ifaces = ",ovsifaces=yes";
        }else{
            $use_ovs_ifaces = "";
        }

        $addr_param = ",addr=" . $mgmt_net["ipaddr"] . "/" . $mgmt_net["cidr"] . ":" . $mgmt_net["gateway"] . ",vmnet_addr=" . $vm_net["ipaddr"] . "/" . $vm_net["cidr"] . ",vmnet_iface0=" . $vm_net["iface1"] . ",vmnet_iface1=" . $vm_net["iface2"]. $use_ovs_ifaces;
        if($mgmt_net["mtu"] != 0){
            $addr_param .= ",mtu=" . $mgmt_net["mtu"];
        }
        $vm_net_vlan = $vm_net["vlan"];
        if(!is_null($vm_net_vlan)){
            $addr_param .= ",vmnet_vlan=$vm_net_vlan";
        }

        $iface1 = get_ethname_from_network_config($con, $conf, $machine_id, $mgmt_net["iface1"], $mgmt_net["lldp_match"]);
        if (is_array($iface1)) { return $iface1; }

        $netvlan = $mgmt_net["vlan"];



        if($mgmt_net["iface2"] != "none"){
            $iface2 = get_ethname_from_network_config($con, $conf, $machine_id, $mgmt_net["iface2"], $mgmt_net["lldp_match"]);
            if (is_array($iface2)) { return $iface2; }
            if(is_null($netvlan)){
                if($cluster["bgp_to_the_host"] == "yes" && $machine["force_no_bgp2host"] == "no"){
                    $mytype = "bgp";
                }else{
                    $mytype = "bond";
                }
                $network_params .= " --static-iface type=$mytype,iface0=$iface1,iface1=$iface2" . $addr_param . $asnum;
            }else{
                if($cluster["bgp_to_the_host"] == "yes" && $machine["force_no_bgp2host"] == "no"){
                    $mytype = "bgp";
                }else{
                    $mytype = "bondvlan";
                }
                $network_params .= " --static-iface type=$mytype,vlannum=$netvlan,iface0=$iface1,iface1=$iface2" . $addr_param . $asnum;
            }
        }else{
            if(is_null($netvlan)){
                $VLAN="";
            }else{
                $VLAN=",vlannum=$netvlan";
            }
            $network_params .= " --static-iface type=normal,".$VLAN."iface0=$iface1" . $addr_param;
        }
        $network_params .= ",extra_ovs=yes";
        if($has_ovs_bridge == "yes"){
            for($i=0;$i<sizeof($ovs_bridge_list);$i++){
                $network_params .= " --static-iface type=ovsbridge,ovsbridgename=" . $ovs_bridge_list[$i]["bridgename"] . ",iface0=" . $ovs_bridge_list[$i]["iface1"] . ",iface1=" . $ovs_bridge_list[$i]["iface2"];
            }
        }
    }else{
        $br_ex_ovs_bridged_network = "no";
        for($i=0;$i<sizeof($machine_networks["networks"]);$i++){
            $onenet  = $machine_networks["networks"][$i];

            if(($onenet["is_public"] == "yes") || ($onenet["role"] == "ovs-bridge")){
                continue;
            }

            // If 2 networks have the same interface names, then we must
            // use virtual interface namings (aka: eth0:0 or bond0:0)
            $virtual_interface_num = 0;
            $use_virtual_interface = "no";
            for($j=0;$j<sizeof($machine_networks["networks"]);$j++){
                if($i == $j){
                    $virtual_interface_name = ":$virtual_interface_num";
                    $virtual_interface_num++;
                    continue;
                }
                if($machine_networks["networks"][$i]['iface1'] == $machine_networks["networks"][$j]['iface1'] && $machine_networks["networks"][$i]['iface2'] == $machine_networks["networks"][$j]['iface2']){
                    $virtual_interface_num++;
                    $use_virtual_interface = "yes";
                }
            }

            $netvlan = $onenet["vlan"];
            if($onenet["role"] == "ceph-cluster"){
                // Do not set gateway for the Ceph cluster network
                $addr_param = ",addr=" . $onenet["ipaddr"] . "/" . $onenet["cidr"] . ":";
            }else{
                $addr_param = ",addr=" . $onenet["ipaddr"] . "/" . $onenet["cidr"] . ":" . $onenet["gateway"];
            }

            $iface1 = get_ethname_from_network_config($con, $conf, $machine_id, $onenet["iface1"], $onenet["lldp_match"]);
            if (is_array($iface1)) { return $iface1; }

            if($onenet["iface2"] != "none"){
                $iface2 = get_ethname_from_network_config($con, $conf, $machine_id, $onenet["iface2"], $onenet["lldp_match"]);
                if (is_array($iface2)) { return $iface2; }
                if(is_null($netvlan)){
                    if($cluster["bgp_to_the_host"] == "yes" && $machine["force_no_bgp2host"] == "no"){
                        $mytype = "bgp";
                    }else{
                        $mytype = "bond";
                    }
                    $network_params .= " --static-iface type=$mytype,iface0=$iface1,iface1=$iface2" . $addr_param . $asnum;
                }else{
                    if($cluster["bgp_to_the_host"] == "yes" && $machine["force_no_bgp2host"] == "no"){
                        $mytype = "bgp";
                    }else{
                        $mytype = "bondvlan";
                    }
                    $network_params .= " --static-iface type=$mytype,vlannum=$netvlan,iface0=$iface1,iface1=$iface2" . $addr_param . $asnum;
                }
            }else{
                if($cluster["nested_openstack"] == "yes" && $machine["brex_macaddr"] != 'none'){
                    $network_params .= " --static-iface type=forcebrexmac,iface0=$iface1,brexmacaddr=" . $machine["brex_macaddr"] . $addr_param;
                }else{
                    if(is_null($netvlan)){
                        $VLAN="";
                    }else{
                        $VLAN=",vlannum=$netvlan";
                    }
                    $network_params .= " --static-iface type=normal".$VLAN.",iface0=$iface1" . $addr_param;
                }
            }
            if(($role == "compute" || $role == "network") && ($onenet["role"] != "ceph-cluster")){
                if($has_ovs_bridge == "no"){
                    $network_params .= ",extra_ovs=yes,ovsbr=br-ex";
                    $br_ex_ovs_bridged_network = "yes";
                }
            }

            if($machine["force_frr"] == "yes"){
                $network_params .= ",forcefrr=yes";
                if($machine["force_frr_nic1"] != "none"){
                    $network_params .= ",forcefrrnic1=".$machine["force_frr_nic1"];
                }
                if($machine["force_frr_nic2"] != "none"){
                    $network_params .= ",forcefrrnic2=".$machine["force_frr_nic2"];
                }
                $network_params .= ",forcefrrvlan=".$machine["force_frr_vlan"].",asnum=" . $cluster["asn"];
            }

            // Setup controller's interface on br-ex if there's compute nodes but no network nodes
            if($role == "controller"){
                // Get the number of compute nodes
                $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='compute'";
                $r = mysqli_query($con, $q);
                if($r === FALSE){
                    $json["status"] = "error";
                    $json["message"] = mysqli_error($con);
                    return $json;
                }
                $num_compute_nodes = mysqli_num_rows($r);

                // Get the number of compute nodes
                $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='network'";
                $r = mysqli_query($con, $q);
                if($r === FALSE){
                    $json["status"] = "error";
                    $json["message"] = mysqli_error($con);
                    return $json;
                }
                $num_network_nodes = mysqli_num_rows($r);
                if($num_compute_nodes > 0 && $num_network_nodes == 0){
                    if($has_ovs_bridge == "no"){
                        $network_params .= ",extra_ovs=yes,ovsbr=br-ex";
                        $br_ex_ovs_bridged_network = "yes";
                    }
                }
            }
            if($onenet["mtu"] != 0){
                $network_params .= ",mtu=" . $onenet["mtu"];
                $br_ex_ovs_bridged_network = "yes";
            }
        }
        if($has_ovs_bridge == "yes"){
            for($i=0;$i<sizeof($ovs_bridge_list);$i++){
                $ovsbridge_iface1 = get_ethname_from_network_config($con, $conf, $machine_id, $ovs_bridge_list[$i]["iface1"]);
                if(is_array($ovsbridge_iface1)) { return $ovsbridge_iface1; }
                $network_params .= " --static-iface type=ovsbridge,ovsbridgename=" . $ovs_bridge_list[$i]["bridgename"] . ",iface0=" . $ovsbridge_iface1;
                if($ovs_bridge_list[$i]["iface2"] != 'none'){
                    $ovsbridge_iface2 = get_ethname_from_network_config($con, $conf, $machine_id, $ovs_bridge_list[$i]["iface2"]);
                    if(is_array($ovsbridge_iface2)) { return $ovsbridge_iface2; }
                    $network_params .= ",iface1=" . $ovsbridge_iface2;
                }
                if(!is_null($ovs_bridge_list[$i]["vlan"])){
                    $network_params .= ",vlannum=" . $ovs_bridge_list[$i]["vlan"];
                }
            }
        }
    }

    // Get its cluster name and fetch the additional package list for this node
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    $package_list_file = "";
    if($n == 1){
        $cluster = mysqli_fetch_array($r);
        $cluster_name = $cluster["name"];
        $package_list_path = "/var/lib/oci/clusters/$cluster_name/" . $machine["hostname"] . "/oci-packages-list";
        if(file_exists($package_list_path)){
            $package_list_file = ",";
            $package_list_file .= file_get_contents($package_list_path);
        }
    }
    if($conf["network"]["debian_keyring_package"] && $conf["network"]["install_debian_keyring_package"]){
        $package_list_file .= "," . $conf["network"]["debian_keyring_package"];
    }

    $cmd  = "oci-install-with-report";
    $cmd .= $network_params;
    $cmd .= " --release ".$conf["releasenames"]["debian_release"];
    $cmd .= " --debootstrap-url ".$conf["network"]["debian_mirror"];
    $cmd .= " --sources.list-mirror ".$conf["network"]["debian_mirror"];
    $cmd .= " --security-mirror ".$conf["network"]["debian_security_mirror"];
    if($conf["network"]["debian_keyring_file"]){
        $cmd .= " --debootstrap-keyring-file " . $conf["network"]["debian_keyring_file"];
        if ($conf["network"]["install_debian_keyring_file"]) {
            $cmd .= " --copy-debootstrap-keyring-file";
        }
    }

    if($machine["boot_uefi"] == "yes"){
        $cmd .= " --boot-type uefi";
    }

    if($machine["install_on_raid"] == "no"){
        if($machine["use_oci_sort_dev"] == "yes"){
            $q = "SELECT * FROM blockdevices WHERE machine_id='".$machine["id"]."' AND name='$install_hdd_name'";
            $r = mysqli_query($con,$q);
            $a = mysqli_fetch_array($r);
            $v = $a["realdev"];
            if(strlen($v) > 0){
                $install_hdd_name = $v;
            }
        }
        $cmd .= " --dest-hdd $install_hdd_name";
    }else{
        $raid_dev0 = $machine["raid_dev0"];
        $raid_dev1 = $machine["raid_dev1"];
        $raid_dev2 = $machine["raid_dev2"];
        $raid_dev3 = $machine["raid_dev3"];
        if($machine["use_oci_sort_dev"] == "yes"){
            $q = "SELECT * FROM blockdevices WHERE machine_id='".$machine["id"]."' AND name='$raid_dev0'";
            $r = mysqli_query($con, $q);
            $a = mysqli_fetch_array($r);
            $v = $a["realdev"];
            if(strlen($v) > 0){
                $raid_dev0 = $v;
            }

            $q = "SELECT * FROM blockdevices WHERE machine_id='".$machine["id"]."' AND name='$raid_dev1'";
            $r = mysqli_query($con, $q);
            $a = mysqli_fetch_array($r);
            $v = $a["realdev"];
            if(strlen($v) > 0){
                $raid_dev1 = $v;
            }

            $q = "SELECT * FROM blockdevices WHERE machine_id='".$machine["id"]."' AND name='$raid_dev2'";
            $r = mysqli_query($con, $q);
            $a = mysqli_fetch_array($r);
            $v = $a["realdev"];
            if(strlen($v) > 0){
                $raid_dev2 = $v;
            }

            $q = "SELECT * FROM blockdevices WHERE machine_id='".$machine["id"]."' AND name='$raid_dev3'";
            $r = mysqli_query($con, $q);
            $a = mysqli_fetch_array($r);
            $v = $a["realdev"];
            if(strlen($v) > 0){
                $raid_dev3 = $v;
            }
        }
        $q = "";
        switch($machine["raid_type"]){
        case "0":
            $cmd .= " --dest-hdd raid0";
            $cmd .= " --raid-devices $raid_dev0,$raid_dev1";
            break;
        case "1":
            $cmd .= " --dest-hdd raid1";
            $cmd .= " --raid-devices $raid_dev0,$raid_dev1";
            break;
        case "10":
            $cmd .= " --dest-hdd raid10";
            $cmd .= " --raid-devices $raid_dev0,$raid_dev1,$raid_dev2,$raid_dev3";
            break;
#        case "5":
#            $cmd .= " --dest-hdd raid5";
#            $cmd .= " --raid-devices ".$machine["raid_dev0"].",".$machine["raid_dev1"].",".$machine["raid_dev2"].",".$machine["raid_dev3"];
#            break;
        default:
            echo "Not implemented yet...";
            die();
            break;
        }
    }
    $cmd .= " --no-cloud-init --extra-packages gnupg2,haveged,uuid-runtime,iotop,iftop,man-db,curl,less,lsb-release,joe,ssl-cert,most,screen,vim,vim-tiny,tcpd,xfsdump,unzip,tcpdump,ca-certificates,lftp,at,tree,lsof,bind9-host,strace,tmux,nano,bash-completion,openssl,file,net-tools,iproute2,ipmitool,ca-certificates,xfsprogs,e2fsprogs,parted,nmap,mtr-tiny$package_list_file";
    $cmd .= " --hook-script /usr/bin/openstack-cluster-installer-bodi-hook-script";
    $cmd .= " --root-ssh-key /root/.ssh/authorized_keys";
    if($machine["cpu_vendor"] == "GenuineIntel"){
        $microcode_pkg = ",intel-microcode";
    }elseif($machine["cpu_vendor"] == "AuthenticAMD"){
        $microcode_pkg = ",amd64-microcode";
    }else{
        $microcode_pkg = "";
    }
    $cmd .= " --install-nonfree-repo --postinstall-packages q-text-as-data,firmware-bnx2,firmware-bnx2x,firmware-linux-free,firmware-misc-nonfree,firmware-qlogic,plymouth,puppet,bridge-utils,grc,ccze,ncdu,lvm2$microcode_pkg,smartmontools,kexec-tools,iptables-persistent,lshw,jq,apg,dmsetup,eperl,gettext-base,lynx,quota,acct,psmisc,chrony";
    if($br_ex_ovs_bridged_network == "yes" && ($role == "compute" || $role == "network" || $role == "controller")){
        $cmd .= ",openvswitch-switch";
    }
    $cmd .= " --hostname " . $machine["hostname"];
    if($machine["serial_console_dev"] != "none"){
        $cmd .= " --add-serial-getty ".$machine["serial_console_dev"];
    }
    $cmd .= " --tty-autologin yes";
    $cmd .= " --reboot-after-install";
    if($role == "swiftstore" || $role == "swiftproxy"){
        if($machine["install_on_raid"] == "no"){
            if($machine["dest_blk"] == "none"){
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da' ORDER BY name";
            }else{
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name != '" . $machine["dest_blk"] . "' AND name != '" . $machine["dest_blk"] . "n1' ORDER BY name";
            }
        }else{
            switch($machine["raid_type"]){
            case "0":
            case "1":
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' ORDER BY name";
                break;
            case "10":
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' AND name NOT LIKE '".$machine["raid_dev2"]."' AND name NOT LIKE '".$machine["raid_dev3"]."' ORDER BY name";
            default:
                echo "Not implemented yet...";
                die();
                break;
            }
        }
        $r = mysqli_query($con, $q);
        $num_block_dev = mysqli_num_rows($r);
        if($num_block_dev < 1 && $role == "swiftstore"){
            $out["status"]  = "error";
            $out["message"] = "Cannot find storage block device for this machine, and it's a swiftstore...<br>";
            return $out;
        }
        if($num_block_dev > 0){
            if($role == "volume"){
                $format_cmd = " --vgcreate ";
            }else{
                $format_cmd = " --xfsformat ";
            }
            for($i=0;$i<$num_block_dev;$i++){
                $blockdev = mysqli_fetch_array($r);
                if($i != 0 && $format_cmd != " --xfsformat "){
                    $format_cmd .= ",";
                }
                if($machine["use_oci_sort_dev"] == "yes"){
                    $qx = "SELECT realdev FROM blockdevices WHERE machine_id='".$machine["id"]."' AND name='".$blockdev["name"]."'";
                    $rx = mysqli_query($con, $qx);
                    $ax = mysqli_fetch_array($rx);
                    $format_cmd .= $ax["realdev"];
                }else{
                    $format_cmd .= $blockdev["name"];
                }
            }
        }else{
            $format_cmd = "";
        }
        $cmd .= $format_cmd;
    }
    if($machine["kernel_from_backports"] == "yes"){
        $cmd .= " --kernel-from-backports yes";
    }
    $cmd .= " >/var/log/oci.log 2>&1 &";

    $out["cmd"] = $cmd;
    return $out;
}

function base64_etc_hosts($con, $conf, $machine_id){
    return base64_encode(slave_calculate_hosts_file($con, $conf, $machine_id));
}

// Calculate a /etc/hosts file containing the IP and names
// of the slave host itself, and all its peers in the cluster
function slave_calculate_hosts_file($con, $conf, $machine_id){
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find machine: $machine_id<br>";
        return $out;
    }
    $machine = mysqli_fetch_array($r);
    $machine_fqdn     = $machine["hostname"];
    $machine_cluster  = $machine["cluster"];
    $machine_ipaddr   = $machine["ipaddr"];
    $machine_role     = $machine["role"];

    if(!is_null($machine_cluster)){
        $q = "SELECT * FROM clusters WHERE id='$machine_cluster'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n != 1){
            $out .= "Cannot find machine: $machine_id<br>";
            return $out;
        }
        $cluster = mysqli_fetch_array($r);
        $cluster_domain       = $cluster["domain"];
        $cluster_id           = $cluster["id"];
        $cluster_name         = $cluster["name"];
        $cluster_vip_hostname = $cluster["vip_hostname"];
        if($cluster_vip_hostname == ""){
            $cluster_vip_hostname = $cluster_name ."-api." . $cluster_domain;
        }

        // Remove the hostname from the machine's FQDN
        $machine_hostname = str_replace("." . $cluster_domain, "", $machine_fqdn);

        $q = "SELECT INET_NTOA(ips.ip) as ipaddr,networks.name AS networkname FROM ips,networks WHERE ips.machine='".$machine["id"]."' AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
#        echo $q;
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n == 1){
            $ip = mysqli_fetch_array($r);
            $machine_ipaddr = $ip["ipaddr"];
        }
    }else{
        $machine_hostname = $machine_fqdn;
    }

    $pxe_server_hostname = gethostname();
    $pxe_server_ipaddr = $conf["network"]["OCI_IP"];

    $out = "# OCISTA_MAINTAINED: Do not touch between these lines, this is a generated content.
127.0.0.1	localhost
$machine_ipaddr	$machine_fqdn $machine_hostname

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

# The puppet master:
$pxe_server_ipaddr	$pxe_server_hostname
";

    $out .= "# VIP address:
";

    // Fetch the API VIP ip
    if(!is_null($machine_cluster)){
        $q = "SELECT * FROM networks WHERE is_public='yes' AND (role='all' OR role='controller' OR role='vip')";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        for($i=0;$i<=$n;$i++){
            $network = mysqli_fetch_array($r);
            $network_id   = $network["id"];
            $network_cidr = $network["cidr"];
            $cluster_id   = $network["cluster"];
            # Fetch the VIP hostname
            $q_clust = "SELECT * FROM clusters WHERE id='$cluster_id'";
            $r_clust = mysqli_query($con, $q_clust);
            $n_clust = mysqli_num_rows($r_clust);
            if($n_clust >= 1){
                $clust = mysqli_fetch_array($r_clust);
                if($clust["vip_hostname"] == ""){
                    $clust_vip_hostname = $clust["name"] . "-api." . $clust["domain"];
                }else{
                    $clust_vip_hostname = $clust["vip_hostname"];
                }
                if($network_cidr == "32"){
                    $vip_addr = $network["ip"];
                    $out .= $vip_addr ." " . $clust_vip_hostname . "\n";
                }else{
                    $qip = "SELECT INET_NTOA(ip) AS addr FROM ips WHERE network='$network_id' AND usefor='vip' AND vip_usage='api'";
                    $rip = mysqli_query($con, $qip);
                    $nip = mysqli_num_rows($rip);
                    if($nip == 1){
                        $vip = mysqli_fetch_array($rip);
                        $vip_addr = $vip["addr"];
                        $out .= $vip_addr ." " . $clust_vip_hostname . "\n";
                    }
                }
                if($clust["haproxy_use_sni"] == "yes"){
                    $service_list = [ "glance", "heat", "heat_cfn", "barbican", "mistral", "vmms", "placement", "nova", "neutron", "ironic", "inspector", "octavia", "designate", "cinder", "gnocchi", "panko", "cloudkitty", "aodh", "magnum", "manila", "keystone" ];
                    foreach ($service_list as &$service) {
                        $out .= $vip_addr ." " . $clust[$service . "_sni_name"] . "." . $clust_vip_hostname . "\n";
                    }
                }
            }
        }
        // Fetch the SQL VIP if it exists
        $q = "SELECT * FROM networks WHERE cluster='$machine_cluster' AND is_public='no' AND role!='ovs-bridge' AND role!='vm-net' AND role!='vip' AND role!='ceph-cluster' AND role!='ipmi' LIMIT 1";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n == 1){
            $network = mysqli_fetch_array($r);
            $network_id   = $network["id"];

            $q = "SELECT INET_NTOA(ip) AS addr FROM ips WHERE network='$network_id' AND usefor='vip' AND vip_usage='sql'";
            $r = mysqli_query($con, $q);
            $n = mysqli_num_rows($r);
            if($n == 1){
                $vip = mysqli_fetch_array($r);
                $vip_addr = $vip["addr"];
                $out .= $vip_addr ." " . $cluster_name ."-sql." . $cluster_domain . "\n";
            }
        }
    }

    $out .= "# Nodes in this cluster:
";
    if(!is_null($machine_cluster)){
        $q = "SELECT * FROM machines WHERE cluster='$machine_cluster' AND id != '$machine_id' ORDER BY role, LENGTH(hostname), hostname";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $other_host = mysqli_fetch_array($r);
            $other_host_ipaddr   = $other_host["ipaddr"];
            $other_host_fqdn     = $other_host["hostname"];
            $other_host_hostname = str_replace("." . $cluster_domain, "", $other_host_fqdn);
            $other_host_id       = $other_host["id"];

            $q_oh = "SELECT INET_NTOA(ips.ip) AS addr FROM networks, ips WHERE ips.machine='$other_host_id' AND networks.cluster='$machine_cluster' AND networks.id=ips.network order by case when role='all' then 1 when role='vm-net'  then 2 when role='ceph-cluster' then 3 else 9 end LIMIT 1";
            $r_oh = mysqli_query($con, $q_oh);
            $n_oh = mysqli_num_rows($r_oh);
            if($n_oh == 1){
                $ip_oh = mysqli_fetch_array($r_oh);
                $other_host_ipaddr = $ip_oh["addr"];
            }
            $out .= "$other_host_ipaddr	$other_host_fqdn $other_host_hostname\n";
        }
    }
    if(file_exists("/etc/openstack-cluster-installer/hosts_append")){
        $out .= file_get_contents("/etc/openstack-cluster-installer/hosts_append");
    }
    $out .= "# OCIFIN_MAINTAINED: Do not touch between these lines, this is a generated content.\n";
    return $out;
}

function build_swift_ring($con, $conf, $cluster_id, $verbose="no", $initial_account_weight=1000, $initial_container_weight=1000, $initial_object_weight=1000, $ec_only="no"){
    $out = "";
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find cluster: $cluster_id<br>";
        return $out;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_name = $cluster["name"];
    $cluster_swift_part_power     = $cluster["swift_part_power"];
    $cluster_swift_replicas       = $cluster["swift_replicas"];
    $cluster_swift_min_part_hours = $cluster["swift_min_part_hours"];

    #################################################
    ### Create the swift ring if it doesn't exist ###
    #################################################
    # First, we check if there's some swiftstore machines in the cluster, in which case
    # we do need a swift ring.

    mkdir_if_not_exists("/var/lib/oci/clusters", 0755);
    mkdir_if_not_exists("/var/lib/oci/clusters/$cluster_name", 0755);

    $swift_ring_path = "/var/lib/oci/clusters/$cluster_name/swift-ring";
    mkdir_if_not_exists($swift_ring_path, 0755);

    if($ec_only == "no"){
        # Account
        $cmd = "swift-ring-builder $swift_ring_path/account.builder create $cluster_swift_part_power $cluster_swift_replicas $cluster_swift_min_part_hours";
        if($verbose == "yes"){ print("Creating account.builder ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Creating account.builder ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);

        # Container
        $cmd = "swift-ring-builder $swift_ring_path/container.builder create $cluster_swift_part_power $cluster_swift_replicas $cluster_swift_min_part_hours";
        if($verbose == "yes"){ print("Creating container.builder ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Creating container.builder ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);

        # Object
        $cmd = "swift-ring-builder $swift_ring_path/object.builder create $cluster_swift_part_power $cluster_swift_replicas $cluster_swift_min_part_hours";
        if($verbose == "yes"){ print("Creating object.builder ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Creating object.builder ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }

    if($cluster["swift_ec_enable"] == "yes"){
        $ec_num_data_fragments   = $cluster["swift_ec_num_data_fragments"];
        $ec_num_parity_fragments = $cluster["swift_ec_num_parity_fragments"];
        $ec_total_num_fragments  = $ec_num_data_fragments + $ec_num_parity_fragments;

        $policy_index = $cluster["swift_ec_policy_index"];
        $ec_num_data_parity = $cluster["swift_ec_policy_index"];

        $cmd = "swift-ring-builder $swift_ring_path/object-".$policy_index.".builder create 10 $ec_total_num_fragments 1";
        if($verbose == "yes"){ print("Creating object-".$policy_index.".builder ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Creating object-".$policy_index.".builder ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }

    $q = "SELECT machines.id AS id FROM ips,machines WHERE machines.cluster='$cluster_id' AND (machines.role='swiftstore' OR machines.role='swiftproxy') AND ips.machine=machines.id ORDER BY ips.ip";
#    if($verbose == "yes"){ print("Selecting: $q<br>\n"); ob_flush(); }else{ $out .= "Selecting: $q\n";}
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $machine = mysqli_fetch_array($r);
	$machine_id = $machine["id"];
        $out .= add_machine_to_swift_ring($con, $conf, $machine_id);
    }

    if($ec_only == "no"){
        # Account
        $cmd = "swift-ring-builder $swift_ring_path/account.builder rebalance";
        if($verbose == "yes"){ print("Rebalancing account ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Rebalancing account ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);

        # Container
        $cmd = "swift-ring-builder $swift_ring_path/container.builder rebalance";
        if($verbose == "yes"){ print("Rebalancing container ring: $cmd<br>"); ob_flush(); }else{ $out .= "Rebalancing container ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);

        # Object
        $cmd = "swift-ring-builder $swift_ring_path/object.builder rebalance";
        if($verbose == "yes"){ print("Rebalancing object ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Rebalancing object ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }

    if($cluster["swift_ec_enable"] == "yes"){
        # EC Object
        $cmd = "swift-ring-builder $swift_ring_path/object-".$policy_index.".builder rebalance";
        if($verbose == "yes"){ print("Rebalancing EC object ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Rebalancing object ring:\n   ===> $cmd\n";}
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }

    $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM ips,machines,networks WHERE networks.id=ips.network AND networks.role!='ipmi' AND networks.role!='ovs-brdige' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ceph-cluster' AND machines.cluster='$cluster_id' AND (machines.role='swiftstore' OR machines.role='swiftproxy') AND ips.machine=machines.id ORDER BY machines.hostname";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    $script = "#!/bin/sh
set -e

";
    for($i=0;$i<$n;$i++){
        $machine = mysqli_fetch_array($r);
        $ipaddr   = $machine["ipaddr"];
        $hostname = $machine["hostname"];
        $script .= "echo \"===> Copying ring to: $hostname\"\n";
        if($cluster["swift_ec_enable"] == "yes"){
            $ec_object_ring = "$swift_ring_path/object-".$policy_index.".builder $swift_ring_path/object-".$policy_index.".ring.gz";
        }else{
            $ec_object_ring = "";
        }
        $script .= "scp $ec_object_ring $swift_ring_path/account.builder $swift_ring_path/container.builder $swift_ring_path/object.builder $swift_ring_path/account.ring.gz $swift_ring_path/container.ring.gz $swift_ring_path/object.ring.gz $ipaddr:/etc/swift\n";
        $script .= "echo \"-> Fixing unix rights\"\n";
        $script .= "ssh $ipaddr \"chown swift:swift /etc/swift/account.ring.gz /etc/swift/container.ring.gz /etc/swift/object.ring.gz\"\n";
    }
    file_put_contents("$swift_ring_path/scp-ring", $script);
    chmod("$swift_ring_path/scp-ring", "0755");

    if($verbose == "yes"){ print("All done!"); }else{ $out .= "All done!";}

    return $out;
}

function add_machine_to_swift_ring($con, $conf, $machine_id, $verbose="no", $initial_account_weight=1000, $initial_container_weight=1000, $initial_object_weight=1000, $ec_only="no"){
    $out = "";

    #################################################
    ### Add a new server to the existing ring.    ###
    #################################################
    # WARNING: if you build a new ring to include new servers,
    # it will break your existing ring (different salt)

    # Gather machine informations
    $q = "SELECT machines.hostname AS hostname, machines.cluster AS cluster_id, INET_NTOA(ips.ip) AS ipaddr, machines.location_id AS locationid, machines.install_on_raid AS install_on_raid, machines.raid_type AS raid_type, machines.raid_dev0 AS raid_dev0, machines.raid_dev1 AS raid_dev1, machines.raid_dev2 AS raid_dev2, machines.raid_dev3 AS raid_dev3, machines.swift_store_account AS swift_store_account, machines.swift_store_container AS swift_store_container, machines.swift_store_object AS swift_store_object FROM ips,machines,networks WHERE networks.id=ips.network AND networks.role!='ipmi' AND networks.role!='ovs-brdige' AND networks.role!='vm-net' AND networks.role!='ceph-cluster' AND networks.role!='vip' AND (machines.role='swiftstore' OR machines.role='swiftproxy') AND ips.machine=machines.id AND machines.id = '$machine_id' LIMIT 1";
#    if($verbose == "yes"){ print("Selecting: $q<br>\n"); ob_flush(); }else{ $out .= "Selecting: $q\n";}
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find machine: $machine_id<br>";
        return $out;
    }
    $machine = mysqli_fetch_array($r);
    $blockdev_locationid = $machine["locationid"];
    $blockdev_ipaddr     = $machine["ipaddr"];
    $hostname            = $machine["hostname"];
    $cluster_id          = $machine["cluster_id"];

    # Gather cluster informations
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find cluster: $cluster_id<br>";
        return $out;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_name = $cluster["name"];

    $swift_ring_path = "/var/lib/oci/clusters/$cluster_name/swift-ring";
    if(!is_dir($swift_ring_path)){
        $out .= "Cannot find ring path $swift_ring_path<br>";
    return $out;
    }

    if($machine["install_on_raid"] == "no"){
        $q = "SELECT blockdevices.name AS hddname FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da' ORDER BY blockdevices.name";
    }else{
        switch($machine["raid_type"]){
        case "0":
        case "1":
            $q = "SELECT blockdevices.name AS hddname FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' ORDER BY blockdevices.name";
            break;
        case "10":
            $q = "SELECT blockdevices.name AS hddname FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' AND name NOT LIKE '".$machine["raid_dev2"]."' AND name NOT LIKE '".$machine["raid_dev3"]."' ORDER BY blockdevices.name";
            break;
        default:
        case "5":
            die("Raid configuratoin for $hostname not supported yet: RAID".$machine["raid_type"]);
            break;
        }
    }
    #if($verbose == "yes"){ print("Selecting: $q<br>\n"); ob_flush(); }else{ $out .= "Selecting: $q\n";}
    $r2 = mysqli_query($con, $q);
    $n2 = mysqli_num_rows($r2);
    $blockdev_object_port = 6200;
    for($j=0;$j<$n2;$j++){
        $blockdev = mysqli_fetch_array($r2);
        $blockdev_devicename = $blockdev["hddname"];

        $q = "SELECT swiftregions.id AS swiftregion FROM locations,swiftregions WHERE locations.id='$blockdev_locationid' AND swiftregions.name=locations.swiftregion";
        $rsw = mysqli_query($con, $q);
        $nsw = mysqli_num_rows($rsw);
        if($nsw == 0){
            $swiftregion = "0";
        }else{
            $asr = mysqli_fetch_array($rsw);
            $swiftregion = $asr["swiftregion"];
        }

        if($machine["swift_store_account"] == "yes" && $ec_only == "no"){
            if(file_exists("$swift_ring_path/account.builder") === FALSE){
                $out .= "No $swift_ring_path/account.builder file found<br>";
                return $out;
            }
            # Account
            $cmd = "swift-ring-builder $swift_ring_path/account.builder add --region $swiftregion --zone $blockdev_locationid --ip $blockdev_ipaddr --port 6002 --device $blockdev_devicename --weight $initial_account_weight";
            if($verbose == "yes"){ print("Adding $hostname, $blockdev_devicename to the account ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Adding $hostname, $blockdev_devicename to the account ring:\n   ===> $cmd\n";}
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);
        }

        if($machine["swift_store_container"] == "yes" && $ec_only == "no"){
            if(file_exists("$swift_ring_path/container.builder") === FALSE){
                $out .= "No $swift_ring_path/container.builder file found<br>";
                return $out;
            }
            # Container
            $cmd = "swift-ring-builder $swift_ring_path/container.builder add --region $swiftregion --zone $blockdev_locationid --ip $blockdev_ipaddr --port 6001 --device $blockdev_devicename --weight $initial_container_weight";
            if($verbose == "yes"){ print("Adding $hostname, $blockdev_devicename to the container ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Adding $hostname, $blockdev_devicename to the container ring:\n   ===> $cmd\n";}
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);
        }

        if($machine["swift_store_object"] == "yes"){
            # Object
            if($ec_only == "no"){
                if(file_exists("$swift_ring_path/object.builder") === FALSE){
                    $out .= "No $swift_ring_path/object.builder file found<br>";
                    return $out;
                }
                $cmd = "swift-ring-builder $swift_ring_path/object.builder add --region $swiftregion --zone $blockdev_locationid --ip $blockdev_ipaddr --port $blockdev_object_port --device $blockdev_devicename --weight $initial_object_weight";
                if($verbose == "yes"){ print("Adding $hostname, $blockdev_devicename to the object ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Adding $hostname, $blockdev_devicename to the object ring:\n   ===> $cmd\n";}
                $output = array();
                $return_var = 0;
                exec($cmd, $output, $return_var);
            }

            if($cluster["swift_ec_enable"] == "yes"){
                if(file_exists("$swift_ring_path/object-".$policy_index.".builder") === FALSE){
                    $out .= "No $swift_ring_path/object-".$policy_index.".builder file found<br>";
                    return $out;
                }
                $cmd = "swift-ring-builder $swift_ring_path/object-".$policy_index.".builder add --region $swiftregion --zone $blockdev_locationid --ip $blockdev_ipaddr --port $blockdev_object_port --device $blockdev_devicename --weight $initial_object_weight";
                if($verbose == "yes"){ print("Adding $hostname, $blockdev_devicename to the erasure coding object ring: $cmd<br>\n"); ob_flush(); }else{ $out .= "Adding $hostname, $blockdev_devicename to the object ring:\n   ===> $cmd\n";}
                $output = array();
                $return_var = 0;
                exec($cmd, $output, $return_var);
            }

            $blockdev_object_port += 1;
        }
    }

    if($verbose == "yes"){ print("Added $machine_id!\n"); }else{ $out .= "Added  $machine_id!\n";}

    return $out;
}

function ipmi_send_command($con, $conf, $machine_id, $command){
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find machine: $machine_id<br>";
        return $out;
    }
    $machine = mysqli_fetch_array($r);

    $ipmi_addr                 = $machine["ipmi_addr"];
    $ipmi_port                 = $machine["ipmi_port"];
    $ipmi_username             = $machine["ipmi_username"];
    $ipmi_password             = $machine["ipmi_password"];

    $cmd = "ipmitool -I lanplus -H $ipmi_addr -p $ipmi_port -U " . escapeshellarg($ipmi_username) . " -P " . escapeshellarg($ipmi_password) . " " .$command;
    $output = "";
    $return_var = 0;
    exec($cmd, $output, $return_var);
    $out = implode("\n", $output);
    return $out;
}

function ipmi_set_boot_device($con, $conf, $machine_id, $bootdev){
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find machine: $machine_id<br>";
        return $out;
    }
    $machine = mysqli_fetch_array($r);

    $ipmi_use                  = $machine["ipmi_use"];
    $ipmi_call_chassis_bootdev = $machine["ipmi_call_chassis_bootdev"];
    $ipmi_addr                 = $machine["ipmi_addr"];
    $ipmi_port                 = $machine["ipmi_port"];
    $ipmi_username             = $machine["ipmi_username"];
    $ipmi_password             = $machine["ipmi_password"];
    $machine_serial            = $machine["serial"];

    if($ipmi_use == "yes" && $ipmi_call_chassis_bootdev == "yes"){
        $cmd = "ipmitool -I lanplus -H $ipmi_addr -p $ipmi_port -U " . escapeshellarg($ipmi_username) . " -P " . escapeshellarg($ipmi_password) . " chassis bootdev $bootdev options=persistent";
        $output = "";
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }
    if( ($machine["product_name"] == "PowerEdge R410") || ($machine["product_name"] == "PowerEdge R610") ){
        if($bootdev == "disk"){
            $cmd = "/opt/dell/toolkit/bin/syscfg --bootseq=hdd.emb.0,nic.emb.1";
        }else{
            $cmd = "/opt/dell/toolkit/bin/syscfg --bootseq=nic.emb.1,hdd.emb.0";
        }
        send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
    }
    if( ($machine["product_name"] == "PowerEdge R740xd") ){
        if($bootdev == "disk"){
            $cmd = "racadm set BIOS.BiosBootSettings.SetBootOrderEn HardDisk.List.1-1,NIC.Integrated.1-1-1";
            send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
            $cmd = "racadm set BIOS.BiosBootSettings.BootSeq HardDisk.List.1-1,NIC.Integrated.1-1-1";
            send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
            $cmd = "racadm set BIOS.BiosBootSettings.HddFailover Enabled";
            send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
            $cmd = "racadm jobqueue create BIOS.Setup.1-1";
            send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
        }else{
            $cmd = "racadm set BIOS.BiosBootSettings.SetBootOrderEn NIC.Integrated.1-1-1,HardDisk.List.1-1";
            send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
            $cmd = "racadm set BIOS.BiosBootSettings.BootSeq NIC.Integrated.1-1-1,HardDisk.List.1-1";
            send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
            $cmd = "racadm jobqueue create BIOS.Setup.1-1";
            send_ssh_cmd($conf, $con, $machine["ipaddr"], $cmd);
        }
    }
    if( $machine["product_name"] == "ProLiant DL385 Gen10 Plus"){
        $cmd = "oci-ilorest $machine_serial bootorder 'HD.EmbSATA.*' --commit";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);

        $cmd = "oci-ilorest $machine_serial bootorder --continuousboot=Hdd --commit";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }
    if( $machine["product_name"] == "ProLiant DL345 Gen11"){
        if($bootdev == "disk"){
            $cmd = "oci-ilorest $machine_serial bootorder 'NVMe.Emb*' --commit";
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);

            $cmd = "oci-ilorest $machine_serial bootorder --continuousboot=Hdd --commit";
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);
        }else{
            $cmd = "oci-ilorest $machine_serial bootorder 'OCP.Slot.*.IPv4' --commit";
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);

            $cmd = "oci-ilorest $machine_serial bootorder --continuousboot=Pxe --commit";
            $output = array();
            $return_var = 0;
            exec($cmd, $output, $return_var);
        }
    }
}

function get_machine_management_network_ip($con, $conf, $machine_id){
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find machine: $machine_id<br>";
        return $out;
    }
    $machine = mysqli_fetch_array($r);
    $cluster_id = $machine["cluster"];
    $location_id = $machine["location_id"];

    $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND location_id='$location_id' AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find network for machine: $machine_id<br>";
        return $out;
    }
    $network = mysqli_fetch_array($r);
    $ret = get_ip_of_machine_on_network($con, $conf, $network["id"], $machine_id);
    return $ret;

}

function gen_oci_facts($con, $conf, $machine_id){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    # Fetch machine
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find machine: $machine_id<br>";
        return $json;
    }
    $machine = mysqli_fetch_array($r);
    $machine_ipaddr   = $machine["ipaddr"];
    $machine_hostname = $machine["hostname"];
    $machine_role     = $machine["role"];
    $cluster_id       = $machine["cluster"];

    # Fetch cluster
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find cluster: $cluster_id<br>";
        return $json;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_name = $cluster["name"];

    $oci_facts = "---
role: $machine_role
cluster: $cluster_name\n";

    # Print out all HDD controllers facts.
    $q = "SELECT * FROM blkdev_ctrl WHERE machine_id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    $oci_facts .= "blkdev_ctrls:\n";
    for($i=0;$i<$n;$i++){
        $a = mysqli_fetch_array($r);
        $oci_facts .= "  - " . $a["product"] . "\n";
    }

    # Add a has_megaraid fact.
    $q = "SELECT * FROM blkdev_ctrl WHERE machine_id='$machine_id' AND ctrl_type='LSI-raid'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n >= 1){
        $oci_facts .= "has_megaraid: true\n";
    }else{
        $oci_facts .= "has_megaraid: false\n";
    }

    # Print out all eth controllers facts.
    $q = "SELECT UNIQUE(driver) FROM ifnames WHERE machine_id='1'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    $oci_facts .= "nic_drivers:\n";
    for($i=0;$i<$n;$i++){
        $a = mysqli_fetch_array($r);
        $oci_facts .= "  - " . $a["driver"] . "\n";
    }

    $json["data"] = $oci_facts;
    return $json;
}

function slave_install_os($con, $conf, $machine_id, $install_cmd){
    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find machine: $machine_id<br>";
        return $out;
    }
    $machine = mysqli_fetch_array($r);
    $machine_ipaddr   = $machine["ipaddr"];
    $machine_hostname = $machine["hostname"];
    $machine_role     = $machine["role"];
    $cluster_id       = $machine["cluster"];

    // Set boot device if using that IPMI option
    ipmi_set_boot_device($con, $conf, $machine_id, "disk");

    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find cluster: $cluster_id<br>";
        return $out;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_name = $cluster["name"];

    #######################################################
    ### Create a folder for filesystem template of host ###
    #######################################################
    $template_path = "/var/lib/oci/clusters/$cluster_name/$machine_hostname";
    mkdir_if_not_exists("/var/lib/oci/clusters", 0755);
    mkdir_if_not_exists("/var/lib/oci/clusters/$cluster_name", 0755);
    mkdir_if_not_exists("/var/lib/oci/clusters/$cluster_name/$machine_hostname", 0755);

    #########################
    ### Manage /etc/hosts ###
    #########################
    // Calculate and scp the /etc/hosts file
    $host_file = slave_calculate_hosts_file($con, $conf, $machine_id);
    file_put_contents("$template_path/oci-hosts-file", $host_file);

    #################################################################
    ### Manage puppet-master hostname file and client certificate ###
    #################################################################
    // Send the puppet-master hostname to /puppet-master-host
    file_put_contents("$template_path/puppet-master-host", gethostname());

    $cmd = "ls /usr/bin/puppetserver";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);
    if($return_var == 0){
        $puppet_master_is_7_or_more = "yes";
    }else{
        $puppet_master_is_7_or_more = "no";
    }

    if($puppet_master_is_7_or_more == "yes"){
        // Delete a probably already existing cert
        $cmd = "sudo /usr/bin/puppetserver ca clean --certname $machine_hostname";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);

        // Generate puppet certificates
        $cmd = "sudo /usr/bin/puppetserver ca generate --certname $machine_hostname";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }else{
        // Delete a probably already existing cert
        $cmd = "sudo /usr/bin/puppet cert clean $machine_hostname";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);

        // Generate puppet certificates
        $cmd = "sudo /usr/bin/puppet ca generate $machine_hostname";
        $output = array();
        $return_var = 0;
        exec($cmd, $output, $return_var);
    }

    $cmd = "sudo /usr/bin/oci-copy-slave-node-generate-key $machine_hostname $template_path 2>&1";
    exec($cmd, $output, $return_var);

    # We keep only the signed certificate, everything else can
    # go away from the puppet-master.
    $cmd = "sudo /usr/bin/oci-remove-slave-node-generated-key $machine_hostname";
    exec($cmd, $output, $return_var);

    #######################################################
    ### Manage PKI x509 slave node certificate and keys ###
    #######################################################
    # Copy all of the CA's .pem files
    $ca_pem_dir = "/var/lib/oci/ssl/ca";
    if (is_dir($ca_pem_dir)) {
        if ($dh = opendir($ca_pem_dir)) {
            while (($file = readdir($dh)) !== false) {
                if(filetype($ca_pem_dir . "/" . $file) == "file"){
                    copy("$ca_pem_dir/$file", "$template_path/$file");
                }
            }
            closedir($dh);
        }
    }

    # Copy the server's private/public keypair
    $node_keys_dir = "/var/lib/oci/ssl/slave-nodes/$machine_hostname";
    if (is_dir($node_keys_dir)) {
        if ($dh = opendir($node_keys_dir)) {
            while (($file = readdir($dh)) !== false) {
                if(filetype($node_keys_dir . "/" . $file) == "file"){
                    copy("$node_keys_dir/$file", "$template_path/$file");
                }
            }
            closedir($dh);
        }
    }

    # If the machine is a controller, then haproxy will need the API SSL keys
    # to be used in haproxy.
    $api_keys = "/var/lib/oci/ssl/slave-nodes/$machine_hostname";
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 1){
        $out .= "Cannot find machine: $machine_id<br>";
        return $out;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_name            = $cluster["name"];
    $cluster_domain          = $cluster["domain"];
    $cluster_vip_hostname    = $cluster["vip_hostname"];
    $first_master_machine_id = $cluster["first_master_machine_id"];
    $cluster_swift_proxy_hostname = $cluster["swift_proxy_hostname"];
    $cluster_self_signed_api_cert = $cluster["self_signed_api_cert"];

    if($cluster_vip_hostname == ""){
        $api_hostname = $cluster_name . "-api." . $cluster_domain;
    }else{
        $api_hostname = $cluster_vip_hostname;
    }

    # Copy the API key
    $api_keys_dir = "/var/lib/oci/ssl/slave-nodes/$api_hostname";
    if (is_dir($api_keys_dir)) {
        if ($dh = opendir($api_keys_dir)) {
            while (($file = readdir($dh)) !== false) {
                if(filetype($api_keys_dir . "/" . $file) == "file"){
                    switch($file){
                    case "$api_hostname.key":
                        // Only controllers need the private key
                        if($machine_role == "controller"){
                            copy("$api_keys_dir/$file", "$template_path/oci-pki-api.key");
                        }
                        break;
                    case "$api_hostname.crt":
                        // But everyone needs the cert
                        copy("$api_keys_dir/$file", "$template_path/oci-pki-api.crt");
                        break;
                    case "$api_hostname.csr":
                        // But everyone needs the cert
                        copy("$api_keys_dir/$file", "$template_path/oci-pki-api.csr");
                        break;
                    case "$api_hostname.pem":
                        // But everyone needs the cert
                        copy("$api_keys_dir/$file", "$template_path/oci-pki-api.pem");
                        break;
                    default:
                        break;
                    }
                }
            }
            closedir($dh);
        }
    }

    mkdir_if_not_exists("$template_path/etc");
    mkdir_if_not_exists("$template_path/etc/oci");
    mkdir_if_not_exists("$template_path/oci-in-target");
    mkdir_if_not_exists("$template_path/oci-in-target/etc");
    mkdir_if_not_exists("$template_path/oci-in-target/etc/oci");

    ###################################
    ### Copy vendor repository keys ###
    ###################################
    mkdir_if_not_exists("$template_path/oci-in-target/etc/oci/vendor-keys");
    $source = '/etc/openstack-cluster-installer/vendor-keys';
    $dest = "$template_path/oci-in-target/etc/oci/vendor-keys";
    if ($handle = opendir($source)) {
        while (($file = readdir($handle)) !== false) {
            // Skip "." and ".."
            if ($file === '.' || $file === '..') {
                continue;
            }

            $srcFile = "$source/$file";
            $destFile = "$dest/$file";

            if (is_file($srcFile)) {
                copy($srcFile, $destFile);
            }
        }
        closedir($handle);
    }

    mkdir_if_not_exists("$template_path/etc/oci/vendor-keys");
    $source = '/etc/openstack-cluster-installer/vendor-keys';
    $dest = "$template_path/etc/oci/vendor-keys";
    if ($handle = opendir($source)) {
        while (($file = readdir($handle)) !== false) {
            // Skip "." and ".."
            if ($file === '.' || $file === '..') {
                continue;
            }

            $srcFile = "$source/$file";
            $destFile = "$dest/$file";

            if (is_file($srcFile)) {
                copy($srcFile, $destFile);
            }
        }
        closedir($handle);
    }

    ##############################
    ### Copy the root CA files ###
    ##############################
    # Note that later on, the openstack-debian-images script will run update-ca-certificates
    # We also transport these through puppet, but it's nicer to have them at setup time.
    mkdir_if_not_exists("$template_path/oci-in-target/usr");
    mkdir_if_not_exists("$template_path/oci-in-target/usr/share");
    mkdir_if_not_exists("$template_path/oci-in-target/usr/share/ca-certificates");
    mkdir_if_not_exists("$template_path/oci-in-target/usr/share/ca-certificates/oci");
    copy("$ca_pem_dir/oci-pki-root-ca.pem", "$template_path/oci-in-target/usr/share/ca-certificates/oci/OCI_1_selfsigned-root-ca.crt");
    copy("$ca_pem_dir/oci-pki-oci-ca.pem", "$template_path/oci-in-target/usr/share/ca-certificates/oci/OCI_2_oci-ca.crt");


    $q = "SELECT * FROM blkdev_ctrl WHERE machine_id='$machine_id' AND product LIKE 'MegaRAID%'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n >= 1){
        file_put_contents("$template_path/oci-in-target/etc/oci/setup-megacli-intarget", "yes");
    }


    ###########################################################
    ### Add a file to tell if the API is self-signed or not ###
    ###########################################################
    if($cluster_self_signed_api_cert == "yes"){
        file_put_contents("$template_path/self-signed-api-cert", "yes");
    }

    #############################
    ### Manage the swift ring ###
    #############################
    $swift_ring_path = "/var/lib/oci/clusters/$cluster_name/swift-ring";

    # Only build the swift ring if there's some swiftstore nodes.
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='swiftstore'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    if($n != 0){
        if(file_exists("$swift_ring_path/account.ring.gz") === FALSE || file_exists("$swift_ring_path/container.ring.gz") === FALSE ||file_exists("$swift_ring_path/object.ring.gz") === FALSE){
            build_swift_ring($con, $conf, $cluster_id);
        }
    }

    if($machine_role == "swiftstore" || $machine_role == "swiftproxy"){
        mkdir_if_not_exists("$template_path/oci-in-target/etc/swift");
        copy("$swift_ring_path/account.ring.gz", "$template_path/oci-in-target/etc/swift/account.ring.gz");
        copy("$swift_ring_path/container.ring.gz", "$template_path/oci-in-target/etc/swift/container.ring.gz");
        copy("$swift_ring_path/object.ring.gz", "$template_path/oci-in-target/etc/swift/object.ring.gz");
        copy("$swift_ring_path/account.builder", "$template_path/oci-in-target/etc/swift/account.builder");
        copy("$swift_ring_path/container.builder", "$template_path/oci-in-target/etc/swift/container.builder");
        copy("$swift_ring_path/object.builder", "$template_path/oci-in-target/etc/swift/object.builder");
        if( file_exists("$swift_ring_path/object-1.builder") === TRUE && file_exists("$swift_ring_path/object-1.ring.gz") === TRUE){
            copy("$swift_ring_path/object-1.builder", "$template_path/oci-in-target/etc/swift/object-1.builder");
            copy("$swift_ring_path/object-1.ring.gz", "$template_path/oci-in-target/etc/swift/object-1.ring.gz");
        }
    }

    # Copy the cert+key if using a custom swiftproxy URL
    if($machine_role == "swiftproxy"){
        if($cluster_swift_proxy_hostname != ""){
            $swift_proxy_key_dir = "/var/lib/oci/ssl/slave-nodes/$cluster_swift_proxy_hostname";
            if (is_dir($swift_proxy_key_dir)) {
                if ($dh = opendir($swift_proxy_key_dir)) {
                    while (($file = readdir($dh)) !== false) {
                        if(filetype($swift_proxy_key_dir . "/" . $file) == "file"){
                            switch($file){
                            case "$cluster_swift_proxy_hostname.key":
                                copy("$swift_proxy_key_dir/$file", "$template_path/oci-pki-swiftproxy.key");
                                break;
                            case "$cluster_swift_proxy_hostname.crt":
                                copy("$swift_proxy_key_dir/$file", "$template_path/oci-pki-swiftproxy.crt");
                                break;
                            case "$cluster_swift_proxy_hostname.csr":
                                copy("$swift_proxy_key_dir/$file", "$template_path/oci-pki-swiftproxy.csr");
                                break;
                            case "$cluster_swift_proxy_hostname.pem":
                                copy("$swift_proxy_key_dir/$file", "$template_path/oci-pki-swiftproxy.pem");
                                break;
                            }
                        }
                    }
                }
            }
        }
    }

    ############################
    ### Manage the /etc/motd ###
    ############################
    if( file_exists("/etc/openstack-cluster-installer/motd") ){
        $motd_content = file_get_contents("/etc/openstack-cluster-installer/motd");
    }else{
        $motd_content = "";
    }

    $motd_content .= "
Welcome to $machine_hostname.
This OS was installed using OCI:
https://salsa.debian.org/openstack-team/debian/openstack-cluster-installer

";
    file_put_contents("$template_path/oci-in-target/etc/motd", $motd_content);

    ############################
    ### Write /etc/oci/my-ip ###
    ############################
    $ret = get_machine_management_network_ip($con, $conf, $machine_id);
    $ip = $ret["data"];
    mkdir_if_not_exists("$template_path/oci-in-target/etc");
    mkdir_if_not_exists("$template_path/oci-in-target/etc/oci");
    file_put_contents("$template_path/oci-in-target/etc/oci/my-ip", $ip);
    file_put_contents("$template_path/oci-in-target/etc/oci/my-role", $machine_role);

    #################################
    ### Write /etc/oci/data-disks ###
    #################################
    if($machine["install_on_raid"] == "no"){
        $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
    }else{
        switch($machine["raid_type"]){
        case "0":
        case "1":
            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."'";
            break;
        case "10":
            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' AND name NOT LIKE '".$machine["raid_dev2"]."' AND name NOT LIKE '".$machine["raid_dev3"]."'";
            break;
        case "5":
        default:
            die("Not supported yet.");
            break;
        }
    }
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    $data_disks = "";
    for($i=0;$i<$n;$i++){
        $a = mysqli_fetch_array($r);
        $hdd_name = $a["name"];
        $data_disks .= "$hdd_name\n";
    }
    file_put_contents("$template_path/oci-in-target/etc/oci/data-disks", $data_disks);

    # Also write if the compute is using hyper-converged model
    if($machine["compute_is_cephosd"] == "yes"){
        file_put_contents("$template_path/oci-in-target/etc/oci/compute_is_cephosd", "yes");
    }

    if($machine_role == "compute"){
        mkdir_if_not_exists("$template_path/oci-in-target/etc/modprobe.d");
        if(($machine["nested_virt"] == "yes") ||  ($machine["nested_virt"] == "cluster_value" && $cluster["nested_virt"] == "yes")){
            file_put_contents("$template_path/oci-in-target/etc/modprobe.d/kvm.conf", "options kvm_intel nested=1");
            file_put_contents("$template_path/oci-in-target/etc/modprobe.d/kvm_amd.conf", "options kvm_amd nested=1");
        }else{
            file_put_contents("$template_path/oci-in-target/etc/modprobe.d/kvm.conf", "options kvm_intel nested=0");
            file_put_contents("$template_path/oci-in-target/etc/modprobe.d/kvm_amd.conf", "options kvm_amd nested=0");
        }
    }

    ####################################################
    ### Write /etc/oci/cinder-separate-volume-groups ###
    ####################################################
    if($machine["cinder_separate_volume_groups"] == "yes"){
        file_put_contents("$template_path/oci-in-target/etc/oci/cinder-separate-volume-groups", "yes");
    }

    #################################################
    ### Add /etc/oci/automatic-ceph-provisionning ###
    #################################################
    if($machine_role == "cephosd" || $machine_role == "billosd" || ($machine_role == "compute" && $machine["compute_is_cephosd"] == "yes")){
        if($cluster["cephosd_automatic_provisionning"] == "yes"){
            file_put_contents("$template_path/oci-in-target/etc/oci/cephosd-automatic-provisionning", "yes");
        }
    }

    ###########################################################
    ### Copy the cluster's ssh keypair if it's a controller ###
    ###########################################################
    if($machine_role == "controller"){
        $ssh_key_dir = "/var/lib/oci/clusters/$cluster_name/ssh";
        if(file_exists("$ssh_key_dir/id_rsa")){
            mkdir_if_not_exists("$template_path/oci-in-target/root");
            mkdir_if_not_exists("$template_path/oci-in-target/root/.ssh", 0700);
            copy("$ssh_key_dir/id_rsa", "$template_path/oci-in-target/root/.ssh/id_rsa");
            chmod("$template_path/oci-in-target/root/.ssh/id_rsa", 0600);
            if(file_exists("$ssh_key_dir/id_rsa.pub")){
                copy("$ssh_key_dir/id_rsa.pub", "$template_path/oci-in-target/root/.ssh/id_rsa.pub");
            }
        }
    }

    #####################################################################
    ### Manage /etc/facter/facts.d default OCI facts like roles, etc. ###
    #####################################################################
    # Please note that what's below is also maintained with puppet
    # in puppet/manifests/generic.pp. Make sure to modify both here
    # and there if you're adding stuff.
    mkdir_if_not_exists("$template_path/oci-in-target/etc/facter");
    mkdir_if_not_exists("$template_path/oci-in-target/etc/facter/facts.d");

    $ret = gen_oci_facts($con, $conf, $machine_id);
    if($ret["status"] != "success"){
        return "Cannot generate OCI fact: " . $ret["message"];
    }
    $oci_facts = $ret["data"];

    # Write the file.
    file_put_contents("$template_path/oci-in-target/etc/facter/facts.d/oci_facts.yaml", $oci_facts);

    ##################################################################
    ### If the machine is a swiftstore, we need the facts.d helper ###
    ##################################################################
    if($machine_role == "swiftstore" || $machine_role == "swiftproxy" || $machine_role == "cephosd"){
        copy("/etc/facter/facts.d/swift_blockdevs_names_to_uuid.sh", "$template_path/oci-in-target/etc/facter/facts.d/swift_blockdevs_names_to_uuid.sh");
        chmod("$template_path/oci-in-target/etc/facter/facts.d/swift_blockdevs_names_to_uuid.sh",0755);
        copy("/etc/facter/facts.d/swift_fstab_dev_list.sh", "$template_path/oci-in-target/etc/facter/facts.d/swift_fstab_dev_list.sh");
        chmod("$template_path/oci-in-target/etc/facter/facts.d/swift_fstab_dev_list.sh",0755);
    }

    #################################################
    ### Copy the gpg pubkey of all backport repos ###
    #################################################
    if(file_exists("/etc/openstack-cluster-installer/oci-repository-key.asc")){
        copy("/etc/openstack-cluster-installer/oci-repository-key.asc", "$template_path/oci-backports-pubkey.gpg");
        copy("/etc/openstack-cluster-installer/oci-repository-key.asc", "$template_path/oci-in-target/etc/oci/oci-repository-key.asc");
        copy("/etc/openstack-cluster-installer/oci-repository-key.asc", "$template_path/etc/oci/oci-repository-key.asc");
    }else{
        if(file_exists("/etc/openstack-cluster-installer/pubkey.gpg")){
            copy("/etc/openstack-cluster-installer/pubkey.gpg", "$template_path/oci-backports-pubkey.gpg");
            copy("/etc/openstack-cluster-installer/pugkey.gpg", "$template_path/oci-in-target/etc/oci/oci-repository-key.asc");
        }
    }

    ####################################################
    ### Create signed SSH host keys for this machine ###
    ### and copy the CA, so we can auth all servers  ###
    ####################################################
    mkdir_if_not_exists("$template_path/oci-in-target/etc");
    mkdir_if_not_exists("$template_path/oci-in-target/etc/ssh");

    # Copy the CA
    copy("/var/lib/oci/clusters/$cluster_name/ssh/ca.pub", "$template_path/oci-in-target/etc/ssh/ca.pub");
    chmod("$template_path/oci-in-target/etc/ssh/ca.pub",0755);

    $ssh_key_dir = "$template_path/oci-in-target/etc/ssh";
    # DSA keys aren't supported in Trixie anymore.
#    # DSA
#    $cmd = "ssh-keygen -P '' -f $ssh_key_dir/ssh_host_dsa_key -t dsa";
#    $output = array();
#    $return_var = 0;
#    exec($cmd, $output, $return_var);

    # ECDSA
    $cmd = "ssh-keygen -P '' -f $ssh_key_dir/ssh_host_ecdsa_key -t ecdsa";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    # ED25519
    $cmd = "ssh-keygen -P '' -f $ssh_key_dir/ssh_host_ed25519_key -t ed25519";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    # RSA
    $cmd = "ssh-keygen -P '' -f $ssh_key_dir/ssh_host_rsa_key -t rsa";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    # Fetch all the IPs assigned to the machine, so we can sign the SSH key for them.
    $q = "SELECT INET_NTOA(ip) AS ipaddr FROM ips WHERE machine='" . $machine["id"] . "' AND usefor='machine'";
    $r = mysqli_query($con, $q);
    $n = mysqli_num_rows($r);
    $all_machine_ips = "";
    for($i=0;$i<$n;$i++){
        $iparray = mysqli_fetch_array($r);
        $all_machine_ips .= "," . $iparray["ipaddr"];
    }

    $machine_short_hostname = strtok($machine_hostname,".");

    # Sign the 4 host keys with the SSH CA
    # DSA keys aren't supported in Trixie anymore.
 #   $cmd = "ssh-keygen -s /var/lib/oci/clusters/$cluster_name/ssh/ca -I $machine_hostname' host key' -n " . $machine_hostname . $all_machine_ips . "," . "localhost" . "," . "127.0.0.1" . "," . $machine_short_hostname . " -V -5m:+3650d -h $ssh_key_dir/ssh_host_dsa_key.pub $ssh_key_dir/ssh_host_ecdsa_key.pub $ssh_key_dir/ssh_host_ed25519_key.pub $ssh_key_dir/ssh_host_rsa_key.pub";
    $cmd = "ssh-keygen -s /var/lib/oci/clusters/$cluster_name/ssh/ca -I $machine_hostname' host key' -n " . $machine_hostname . $all_machine_ips . "," . "localhost" . "," . "127.0.0.1" . "," . $machine_short_hostname . " -V -5m:+3650d -h $ssh_key_dir/ssh_host_ecdsa_key.pub $ssh_key_dir/ssh_host_ed25519_key.pub $ssh_key_dir/ssh_host_rsa_key.pub";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    # Make the ssh_known_hosts to include the CA
    $ssh_known_hosts = "@cert-authority * " . file_get_contents("/var/lib/oci/clusters/$cluster_name/ssh/ca.pub");
    file_put_contents("$ssh_key_dir/ssh_known_hosts", $ssh_known_hosts);

    ###########################################################################
    ### Create a tarball from the $template_path folder, scp it, extract it ###
    ###########################################################################
    $cmd = "cd /var/lib/oci/clusters/$cluster_name/$machine_hostname && tar -C /var/lib/oci/clusters/$cluster_name/$machine_hostname -cvzf /var/lib/oci/clusters/$cluster_name/$machine_hostname.tar.gz *";
    $output = array();
    $return_var = 0;
    exec($cmd, $output, $return_var);

    scp_a_file($conf, $con, $machine_ipaddr, "/var/lib/oci/clusters/$cluster_name/$machine_hostname.tar.gz", "/oci-to-extract-tarball.tar.gz", 0644);
    $ret = send_ssh_cmd($conf, $con, $machine_ipaddr, "tar -C / -xvzf /oci-to-extract-tarball.tar.gz");

    #####################################
    ### Perform the actual OS install ###
    #####################################
    $ret = send_ssh_cmd($conf, $con, $machine_ipaddr, $install_cmd);
}

function oci_install_machine($con, $conf, $machine_id){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    $q = "SELECT * FROM machines WHERE id='$machine_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find machine in database when trying to install in oci_install_machine().";
        return $json;
    }
    $machine = mysqli_fetch_array($r);
    $safe_machine_serial = $machine["serial"];

    if($machine["status"] != "live"){
        $json["status"] = "error";
        $json["message"] = "Error: machine with serial $safe_machine_serial isn't in status live.";
        return $json;
    }

    if(is_null($machine["cluster"])){
        $json["status"] = "error";
        $json["message"] = "Error: machine with serial $safe_machine_serial isn't part of a cluster.";
        return $json;
    }

    if($machine["role"] == "controller"){
        # If we're installing a controller, then obviously, we must setup db and accounts from scratch,
        # so let's set initial_cluster_setup to yes if that's a controller.
        $r = mysqli_query($con, "UPDATE clusters SET initial_cluster_setup='yes' WHERE id='".$machine["cluster"]."'");
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con);
            return $json;
        }
    }
    # When a machine gets reinstalled, obviously, puppet hasn't run yet there...
    $r = mysqli_query($con, "UPDATE machines SET puppet_status='notrun' WHERE id='$machine_id'");
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con);
        return $json;
    }

    $slave_install_return = slave_install_server_os_command($con, $conf, $machine_id);
    if($slave_install_return["status"] != "success"){
        $json["status"] = "error";
        $json["message"] = "Error while calculating installation command line for host $safe_machine_serial: ".$slave_install_return["message"];
        return $json;
    }
    $json["data"] = "Running: ". $slave_install_return["cmd"];
    slave_install_os($con, $conf, $machine_id, $slave_install_return["cmd"]);

    return $json;
}

?>
