# Copyright (c) 2018-2019 VMware, Inc.  All rights reserved.
# -- VMware Confidential


#
# pyHbr.vmfactory.py
#
# Python wrappers for creating running, churning VMs
#
# The only real exported entrypoint is CreateChurningVM()
#

import os
import sys
import subprocess
import tempfile
import shutil
import string

import pyVim.vm
import pyVim.vmconfig

from pyHbr import ovflib
from pyHbr.disks import ParseDatastorePath
from pyVmomi import Vmodl
from pyVmomi import Vim # Primary-side VMODL + Hostd VMODL
from pyVmomi import Hbr # Replica-side VMODL


# vmxCfgTemplate -
#
# This is the template for the 'newtinyvm' VM configuration.
#
# Define at top-level to side-step indentation issues
# XXX PR1404871 Need to update hardware/tinyvm linux version
vmxCfgTemplate="""\
#!/bin/vmx
#
# This VMX configuration generated by hbrvmcfg.py
# ${comment}
#
.encoding = "UTF-8"
config.version = "8"
virtualHW.version = "12"
memsize = "${memsize}"

displayName = "${displayName}"
guestOS = "linux"

# Save output from serial port in a file
serial0.present = "true"
serial0.yieldOnMsrRead = "true"
serial0.fileType = "file"
serial0.fileName = "${serialLog}"
answer.msg.serial.file.open = "Replace"

# Disable VM stats gathering (on debug builds)
statslog.temporaryFileName = "/dev/null"

# No ethernet, no USB, no IDE ...
ethernet0.present = "FALSE"
usb.present = "FALSE"
ide0:0.present = "FALSE"

# Allow unmaps
disk.scsiUnmapAllowed = "TRUE"

# Bump SCSI emulation version to SPC4
disk.scsiversion.spc4 = "TRUE"

# Allow VM poweron on nested ESX
vmx.allownested = "TRUE"
"""


def ExpandTtyLinux(tgzpath):
   """
   Expand the given ttylinux .tgz into a temporary directory on the local
   filesystem.

   Returns the directory created and populated.
   """
   extractDir = tempfile.mkdtemp(prefix="hbrvmcfg.")

   print "Extracting", tgzpath

   rc = subprocess.call(["tar",
                         #"--verbose",
                         "--extract",
                         "--gunzip",
                         "-C", extractDir,
                         "--file", tgzpath])
   if rc != 0:
      raise RuntimeError, "Failed to extract %s to %s: %d" % (tgzpath, extractDir, rc)

   return extractDir


def FindTtyLinux():
   """
   Find the test-esx ttylinux .tar.gz image.

   Returns the complete path to the test-esx ttylinux tar.gz image
   """
   vmtree = os.getenv("VMTREE")
   if not vmtree:
      raise RuntimeError, "VMTREE must be set to find the ttylinux image."

   lin = os.path.join(vmtree, "vmkernel/tests/data/test-esx-ttylinux.tar.gz")

   if not os.path.exists(lin):
      raise RuntimeError, "Could't find ttylinux '%s'" % lin

   return lin

def FindMemTestIso():
   """
   Find the memtest.iso image

   Returns the complete path to the test-esx ttylinux tar.gz image
   """
   vmtree = os.getenv("VMTREE")
   if not vmtree:
      raise RuntimeError, "VMTREE must be set to find the iso image."

   lin = os.path.join(vmtree, "vmkernel/tests/isos/memtest86-3.2.iso")

   if not os.path.exists(lin):
      raise RuntimeError, "Could't find memtest86-3.2.iso '%s'" % lin

   return lin


def FindChurnScript():
   """
   Find the 'diskchurn2.sh' script

   Returns the complete path to the diskchurn2.sh script
   """
   vmtree = os.getenv("VMTREE")

   if not vmtree:
      raise RuntimeError, "VMTREE must be set to find the diskchurn2.sh script."

   script = os.path.join(vmtree, "support/scripts/diskchurn2.sh")

   if not os.path.exists(script):
      raise RuntimeError, "Could't find churn script '%s'" % script

   return script


def FindBlastScript():
   """
   Find the 'diskblast.sh' script

   Returns the complete path to the diskblast.sh script
   """
   vmtree = os.getenv("VMTREE")

   if not vmtree:
      raise RuntimeError, "VMTREE must be set to find the diskblast.sh script."

   script = os.path.join(vmtree, "support/scripts/diskblast.sh")

   if not os.path.exists(script):
      raise RuntimeError, "Could't find blast script '%s'" % script

   return script


def FindMkFloppy():
   """
   Find the 'mkfloppy' script.

   Returns the complete path to the script.
   """
   vmtree = os.getenv("VMTREE")
   if not vmtree:
      raise RuntimeError, "VMTREE must be set to find the ttylinux mkflopy script"

   return os.path.join(vmtree, "support/scripts/mkfloppy-fat.sh")


def FindSg3Utils():
   """
   Find the paths for the sg3utils

   Returns the path to the sg3 binary directory
   """

   toolPath="/build/toolchain/lin32/sg3_utils-1.32/bin"

   if not os.path.exists(toolPath):
      raise RuntimeError, "Couldn't find sg3 util directory '%s'" % toolPath

   return toolPath


def FindChurnExtras():
   """
   Find the paths to the tools that are needed by the churn script.

   Returns a list of paths to tools that the churn run script needs
   """

   utils=["sg_vpd", "sg_unmap", "sg_dd", "sg_readcap", "sg_write_same", "sg_inq"]
   utilsFullPath=[os.path.join(FindSg3Utils(), x) for x in utils]

   for u in utilsFullPath:
      if not os.path.exists(u):
         raise RuntimeError, "Couldn't find sg3 util '%s'" % u

   return utilsFullPath


def FindBlastExtras():
   """
   Find the paths to the tools that are needed by the disk blast script.

   Returns a list of paths to tools that the churn run script needs
   """

   utils=["sg_vpd", "sg_unmap", "sg_dd", "sg_readcap"]
   utilsFullPath=[os.path.join(FindSg3Utils(), x) for x in utils]

   for u in utilsFullPath:
      if not os.path.exists(u):
         raise RuntimeError, "Couldn't find sg3 util '%s'" % u

   return utilsFullPath


def CreateVMXConfigFile(vmx,
                        vmdk,
                        memsize=128,
                        displayName="Unspecified VM display name",
                        nvram="non-specific.nvram",
                        serialLog="/dev/null",
                        floppyName="/dev/null",
                        extraDisks=None,
                        comment=None,
                        diskless=False,
                        isoName="unspecified"):
   """
   Create a customized VMX config file from the vmxCfgTemplate template.

   @param vmx [in]         name of the vmx config file to create
   @param vmdk [in]        name of the boot vmdk containing the ttylinux image
   @param memsize [in]     memsize of the vm (optional)
   @param displayName [in] display name of the vm (optional)
   @param nvram [in]       name of the nvram file to hook the VM up to
   @param serialLog [in]   name of the serial log ttylinux should write to
   @param floppyName [in]  name of the boot floppy image
   @param extraDisks [in]  array of vmdks to attach as add'l scsi disks
   @param comment [in]     An optional comment string to include at the top
   """

   comment = comment.replace("\n", "\n#")

   s = string.Template(vmxCfgTemplate)
   cfg = open(vmx, "w")
   cfg.write(s.substitute(memsize=memsize,
                          displayName=displayName,
                          serialLog=serialLog,
                          comment=comment))

   scsiDev=0
   diskID=1 # boot disk is at scsi0:0
   for xVmdk in extraDisks:
      # Filled the last device, so add a new device for this disk
      if diskID == 16:
         diskID = 0
         scsiDev = scsiDev + 1

         cfg.write("\n")
         cfg.write("### Need another scsi device for this many disks.\n")
         cfg.write("scsi%d.present=\"true\"\n" % scsiDev)
         cfg.write("scsi%d.virtualDev=\"lsilogic\"\n" % scsiDev)
         cfg.write("\n")

      cfg.write('scsi%d:%d.present  = "true"\n' % (scsiDev, diskID))
      cfg.write('scsi%d:%d.fileName = "%s"\n' % (scsiDev, diskID, xVmdk))
      cfg.write('scsi%d:%d.deviceType = "scsi-hardDisk"\n' % (scsiDev, diskID))
      cfg.write('\n')

      diskID = diskID + 1
      if diskID == 7:
         #
         # SCSI ID 7 is the automatic SCSI ID for the HBA.
         # So we need to skip it.
         #
         diskID = 8

   if diskless:
      cfg.write('ide1:0.deviceType = cdrom-image\n')
      cfg.write('ide1:0.fileName = %s\n' % isoName)
      cfg.write('ide1:0.present = TRUE\n')
      cfg.write('ide1:0.startConnected = TRUE\n')
      cfg.write('floppy0.present = FALSE\n')
   else:
      cfg.write('nvram = "%s"\n' % nvram)
      cfg.write('ide1:0.present = FALSE\n')
      cfg.write('scsi0.present = "true"\n')
      cfg.write('scsi0.virtualDev = "lsilogic"\n')

      cfg.write('scsi0:0.present = "true"\n')
      cfg.write('scsi0:0.fileName = "%s"\n' % vmdk)
      cfg.write('scsi0:0.deviceType = "scsi-hardDisk"\n')

      # Connect a floppy disk.  VM will read scripts from here.
      cfg.write('floppy0.startConnected = "true"\n')
      cfg.write('floppy0.clientDevice = "FALSE"\n')
      cfg.write('floppy0.fileType = "file"\n')
      cfg.write('floppy0.fileName = "%s"\n' % floppyName)


   # Let the VM run on legacy CPU. Can be removed once we completely swith
   # to newer hosts
   cfg.write('monitor.allowLegacyCPU = true')

   cfg.write("###\n")
   cfg.write("### Config options added at run-time:\n")
   cfg.write("###\n")
   cfg.close()


def GetRelativePath(file, path):
   """Return a relative path

   @param file [in]      Full path to a file
   @param path [in]      A path
   @return relPath [out] The relative path from path to file
   """
   import re
   relPath = re.sub(path, '', file)
   if relPath.startswith('/'):
      return relPath.lstrip('/')
   return relPath


def CopyFiles(localdir, host, datastore, destDir, user='root', pwd=''):
   """
   Copy all files from the given 'localdir' into the given destination
   directory on the given host's datastore.

   @param localdir [in]  local path to copy (all files) from
   @param host [in]      host to copy to
   @param datastore [in] datastore to copy to
   @param destDir [in]   directory within datastore to copy to (this dir must
                         already exist on the filesystem)
   @param user [in]      the username of the host to copy to
   @param pwd [in]       the password of the host to copy to
   """
   try:
      import paramiko

      transport = paramiko.Transport((host, 22))
      transport.connect(username=user, password=pwd)
      sftp = paramiko.SFTPClient.from_transport(transport)

      destRoot ="/vmfs/volumes/%s/%s" % (datastore, destDir)
      print "Copying VM files to %s" % destRoot

      try:
         for root, dirs, files in os.walk(localdir):
            for dir in dirs:
               currentdir = os.path.join(root, dir)
               relDirPath = GetRelativePath(currentdir, localdir)
               remotedir = os.path.join(destRoot, relDirPath)
               sftp.mkdir(remotedir)
            for file in files:
               currentfile = os.path.join(root, file)
               relFilePath = GetRelativePath(currentfile, localdir)
               remotefile = os.path.join(destRoot, relFilePath)
               sftp.put(currentfile, remotefile)
         sftp.close()
         transport.close()
      except Exception, e:
         sftp.close()
         transport.close()
         raise e
   except ImportError:
      # We couldn't import paramiko, so fall back to scp
      scpSrc="'"+localdir+"'/*" # shell glob, run scp under sh
      scpDest="'root@"+host+":/vmfs/volumes/"+datastore+"/"+destDir+"/'"

      print "Copying VM files to", scpDest
      cmd = ["/bin/sh",
             "-c",
             "scp -p %s %s" % (scpSrc, scpDest)]
      rc = subprocess.call(cmd)
      if rc != 0:
         print "CMD", ' '.join(cmd)
         raise RuntimeError, "scp to '%s' failed: %d" % (host, rc)


def DevStrToDiskNum(label):
   """
   Convert a linux scsi device label (the X in /dev/sdX) into a disk
   number.  Its basically a base-26 number using a-z as the digits.
   """
   l = map(lambda x: ord(x) - ord('a'), list(label))
   if len(l) == 1:
      val = l[0]
   else:
      assert len(l) == 2
      val = (l[0] * 26) + l[1]

   return val


def DevMajor(label):
   # See http://lxr.linux.no/#linux+v2.6.34/Documentation/devices.txt
   scsiMajors = [ 8, 65, 66, 67, 68, 69, 70, 71 ]
   # Minors are 0, 16, 32, ..., (so 16 disks per major)
   diskNum = DevStrToDiskNum(label)
   return scsiMajors[diskNum / 16]


def DevMinor(label):
   # See http://lxr.linux.no/#linux+v2.6.34/Documentation/devices.txt
   # Minors are 0, 16, 32, ..., (so 16 disks per major)
   diskNum = DevStrToDiskNum(label)
   return (diskNum % 16) * 16


def CreateTestVmFloppy(localdir, floppy, command, files, devLabels):
   """
   Create a floppy image designed for driving the ttylinux VM.

   Include any optional FILES in floppy image.

   Generate a boot-strap script, too.  By default it will do 'command'
   which should be a fragment of shell script.  Otherwise, the script will
   execute the first file in the files list.  Barring that, it will just
   sleep forever.

   Generate code to fixup device nodes (/dev/sd*) to access any disks that
   aren't already included on the ttylinux disk image (based on the
   'devLabels' parameter).

   Returns the command string that will be embedded in the generated
   startup script.  (Generally just the 'command' argument, unless that is
   None.)
   """

   if 'test-esx-ttylinux.sh' in files:
      raise RuntimeError, "Cannot include your own test-esx-ttylinux.sh script"

   mkfloppy = FindMkFloppy()

   # Write the shell script that will run in the VM.
   # NOTE: the script name is fixed (the ttylinux VM looks for that name).
   script=os.path.join(localdir, "test-esx-ttylinux.sh")
   f=open(script, 'w')
   if not files and not command:
      command="while true; do sleep 1000000; done;"
   elif not command:
      command="exec %s" % os.path.basename(files[0])

   f.write('#!/bin/sh\n')
   # "/test-esx" is the directory the ttylinux VM will mount the floppy on
   f.write('export PATH="${PATH}:/test-esx"\n')
   f.write('echo "BEGIN -- I am this script:"\n')
   f.write('cat $0\n')
   f.write('echo "END -- I am this script:"\n')
   # Generate fixup code for the missing scsi dev nodes:
   for l in devLabels:
      f.write('[ -b /dev/sd%s ] || mknod /dev/sd%s b %u %u\n' % \
              (l, l, DevMajor(l), DevMinor(l)))
   f.write('%s\n' % command)
   f.write('#eof\n')
   f.close()

   rc = subprocess.call([mkfloppy, floppy, script] + files)
   if rc != 0:
      raise RuntimeError, "mkfloppy %s failed: %d" % (floppy, rc)

   # Copied into floppy, so don't need this anymore.
   os.remove(script)

   return command


def DevLabelsFor(disks, skipDisks=[]):
   """
   Make a list of device labels (e.g., the XX in /dev/sdXX) for the given
   list of disks.  Skips over /dev/sda.

   Used to generate a diskchurn script for all the attached disks.
   """
   pfx=["", "a", "b", "c", "d" ] # Only 128 devices are supported by Linux
   devnames = {
      '': range(ord('b'), ord('z') + 1),
   }
   devnames.update({ch: range(ord('a'), ord('z') + 1)
                    for ch in ['a', 'b', 'c', 'd']})
   devnames = ['{}{}'.format(pfx, chr(letter))
               for pfx, letters in devnames.iteritems()
               for letter in letters]

   result = []
   for i in range(len(disks)):
      if i in skipDisks:
         continue

      result += [devnames[i]]

   return result


def CreateChurningVM(priHostd,
                     vmname, datastore, destDir,
                     scorch=False,
                     stdchurn=False,
                     churnrate=None,
                     disks=[],
                     cpfiles=[],
                     script=None,
                     stdblast=False,
                     minimal=True,
                     baseDiskType=Vim.VirtualDiskManager.VirtualDiskType.thin,
                     snapDepth=0,
                     skipDisks=[]):
   """
   Common routine for creating a 'churn' VM and configuring it to run a
   simple startup script.

   @param[in] priHostd primary hostd connection object
   @param[in] vmname The name of the VM (used for displayname and vmx config file name, too)
   @param[in] datastore The name of the datastore to put the VM's files on (in priHostd)
   @param[in] destDir The relative pathname of the directory into which the VM's files will be dumped
   @param[in] scorch A bool indicating if any existing destDir should be removed
   @param[in] stdchurn A bool indicating if a "standard" churn configuration should be generated
   @param[in] churnrate A string "KB/min[,KBkeep][,RandomY/N][,Unmap %][,Compress %]" describing the argument to diskchurn2.sh
   @param[in] disks An (optional) list of disks (given by a string size in GB, can be fractional) to attach to VM
   @param[in] cpfiles An (optional) list of arbitrary files to copy into a fixed directory in the VM
   @param[in] script An (optional) script to run
   @param[in] stdblast A bool indicating if a "standard" disk blast configuration should be generated
   @param[in] minimal Use the minimum amount of VM configuration information.
   @param[in] baseDiskType The disk type to use when creating the base disk.
   @param[in] snapDepth Make each disk a child disk this number of snapshots deep.

   Optional parameters may be none.

   The disks specified are in addition to the 10MB base disk created for the VM.
   """

   vm = priHostd.FindVmByName(vmname)
   if vm:
      print "VM with name '%s' already exists.  Removing." % vmname
      # make this a pyVim.vm.VM
      churnvm = priHostd.GetPyVimVM(vm)

      if churnvm.IsRunning():
         churnvm.PowerOff()
      churnvm.Unregister()

   # XXX make sure vmname, datastore, directory are sane/simple

   print "Creating directory", destDir, "on", datastore, "..."

   # Look up the datastore (and connect to hostd as a side-effect)
   ds = priHostd.RemoteStorage(datastore)

   # Make a directory for the new VM on the ESX datastore
   try:
      ds.MakeDirectory(destDir)
   except Vim.Fault.FileAlreadyExists, e:
      if scorch:
         print "  Deleting existing", destDir
         ds.CleanupDirectory(destDir)
         ds.MakeDirectory(destDir) # this better not fail..
      else:
         raise RuntimeError, "Directory %s already exists " + \
               "(add --scorch to delete it): %s" \
               % (destDir, str(e))

   if not disks:
      disks = []

   devLabels = DevLabelsFor(disks, skipDisks)

   if stdchurn and stdblast:
      raise RuntimeError, "Can't do both churn and blast at the same time."

   if stdchurn or stdblast:
      # Add the parameters (if necesssary) to make a simple churn VM
      # A 1GB disk, the 'diskchurn.sh' script, and a command line that
      # runs diskchurn
      if not disks:
         disks = [ "1", ] # one GB disk
         devLabels = DevLabelsFor(disks)

      if stdchurn:
         cpfiles = cpfiles + [ FindChurnScript() ] + FindChurnExtras()
      else:
         cpfiles = cpfiles + [ FindBlastScript() ] + FindBlastExtras()

      if not script:
         if not churnrate:
            churnrate = ""

         churnInArgs=churnrate.split(",")
         churnOutArgs=""

         if stdchurn:
            #
            # Parse the churn arguments.
            #
            if len(churnInArgs) >= 1 and churnInArgs[0]:
               churnOutArgs="%s -c %s" % (churnOutArgs, churnInArgs[0])

            if len(churnInArgs) >= 2 and churnInArgs[1]:
               churnOutArgs="%s -s %s" % (churnOutArgs, churnInArgs[1])

            if len(churnInArgs) >= 3 and churnInArgs[2]:
               if churnInArgs[2] == "y" or churnInArgs[2] == "Y":
                  churnOutArgs="%s -r" % churnOutArgs

            if len(churnInArgs) >= 4 and churnInArgs[3]:
               churnOutArgs="%s -u %s" % (churnOutArgs, churnInArgs[3])

            if len(churnInArgs) >= 5 and churnInArgs[4]:
               churnOutArgs="%s -k %s" % (churnOutArgs, churnInArgs[4])

            if len(churnInArgs) >= 6 and churnInArgs[5]:
               churnOutArgs="%s -w %s" % (churnOutArgs, churnInArgs[5])

            if len(churnInArgs) >= 7 and churnInArgs[6]:
               churnOutArgs="%s -y %s" % (churnOutArgs, churnInArgs[6])

            script = 'echo "Automated Disk Churnery:"\n' + \
                     'export PATH=".:${PATH}"\n' + \
                     'for disk in %s; do\n' % ' '.join(devLabels) + \
                     '  usleep 100\n' + \
                     '  diskchurn2.sh %s /dev/sd${disk} &\n' % (churnOutArgs) + \
                     'done\n' + \
                     'sleep 1\n' + \
                     'wait\n'
         else:
            #
            # Parse the blast arguments
            #
            if len(churnInArgs) >= 1 and churnInArgs[0]:
               if churnInArgs[0] == "y" or churnInArgs[0] == "Y":
                  churnOutArgs="%s -r" % churnOutArgs

            if len(churnInArgs) >= 2 and churnInArgs[1]:
               churnOutArgs="%s -u %s" % (churnOutArgs, churnInArgs[1])

            if len(churnInArgs) >= 3 and churnInArgs[2]:
               churnOutArgs="%s -k %s" % (churnOutArgs, churnInArgs[2])

            script = 'echo "Automated Disk Blastery:"\n' + \
                     'export PATH=".:${PATH}"\n' + \
                     'for disk in %s; do\n' % ' '.join(devLabels) + \
                     '  usleep 100\n' + \
                     '  diskblast.sh %s /dev/sd${disk} &\n' % (churnOutArgs) + \
                     'done\n' + \
                     'sleep 1\n' + \
                     'wait\n'

   # Make any extra (blank) disks if necessary
   extraVmdks=[]
   index=0
   for s in disks:
      vmdkName="extraDisk-%03d.vmdk" % index
      sizeKb=long(float(s) * 1024 * 1024)
      print "Creating extra disk", vmdkName, ";", sizeKb, "(kb)"

      hier = ds.CreateDiskHierarchy(basePath=os.path.join(destDir, vmdkName),
                                    numSnaps=snapDepth, sizeKb=sizeKb,
                                    diskType=baseDiskType)
      # Use the child-most disk.
      extraVmdks.append(os.path.basename(hier[-1]))
      index = index + 1

   # Create the VM image locally
   localdir = ExpandTtyLinux(FindTtyLinux())

   vmx = "%s.vmx" % vmname
   basevmdk = "test-esx-tinylinux.vmdk" # defined inside ttylinux .tgz
   nvram = "ttylinux.nvram" # part of ttylinux .tgz

   if minimal:
      #
      # A truly minimal VM doesn't have an nvram file already
      # created, just the VMX file. So for a truly minimal VM
      # we need to ditch the nvram file. However it's perfectly
      # ok to write the name to the config file so it has a
      # reasonable name when it's created.
      #
      # So just remove the prefab nvram before we copy anything.
      #
      os.remove(os.path.join(localdir, nvram))

   floppy = "%s.floppy" % vmname

   # Create the ttylinux boot floppy image
   cmd = CreateTestVmFloppy(localdir, os.path.join(localdir, floppy),
                            script, cpfiles, devLabels)

   # Create the VM's .vmx config file
   CreateVMXConfigFile(vmx=os.path.join(localdir, vmx),
                       vmdk=basevmdk,
                       nvram=nvram,
                       displayName="%s" % vmname,
                       serialLog="vm-serial-output.txt",
                       floppyName=floppy,
                       extraDisks=extraVmdks,
                       comment="Generated floppy will run: %s" % cmd)

   # Copy everything up to the host
   CopyFiles(localdir,
             priHostd.Hostname(),
             datastore,
             destDir,
             priHostd.Username(),
             priHostd.Password())

   # Clean up the local copy of the VM
   shutil.rmtree(localdir)

   print "Created virtual machine", vmx

   # Register the VM on the current hostd
   cfgPath = ds.DatastorePath(os.path.join(destDir, vmx))
   try:
      tsk = priHostd.RegisterVm(cfg=cfgPath, name=vmname)
      priHostd.WaitForTask(tsk)
   except Vim.Fault.AlreadyExists, e:
      print "(VM %s (%s) already registered: %s)" % (vmname, cfgPath, str(e))

   vm = priHostd.FindVmByName(vmname)
   if not vm:
      raise RuntimeError, "Cannot find newly created vm %s" % vmname

   return priHostd.GetPyVimVM(vm)

def CreateDisklessVM(priHostd,
                     vmname, datastore, destDir,
                     scorch=False):
   """
   Common routine for creating a diskless VM. The VM just runs the x86 memtest.

   @param[in] priHostd primary hostd connection object
   @param[in] vmname The name of the VM (used for displayname and vmx config file name, too)
   @param[in] datastore The name of the datastore to put the VM's files on (in priHostd)
   @param[in] destDir The relative pathname of the directory into which the VM's files will be dumped
   @param[in] scorch A bool indicating if any existing destDir should be removed
   """

   vm = priHostd.FindVmByName(vmname)
   if vm:
      print "VM with name '%s' already exists.  Removing." % vmname
      # make this a pyVim.vm.VM
      oldvm = priHostd.GetPyVimVM(vm)

      if oldvm.IsRunning():
         oldvm.PowerOff()
      oldvm.Unregister()

   # XXX make sure vmname, datastore, directory are sane/simple

   print "Creating directory", destDir, "on", datastore, "..."

   # Look up the datastore (and connect to hostd as a side-effect)
   ds = priHostd.RemoteStorage(datastore)

   # Make a directory for the new VM on the ESX datastore
   try:
      ds.MakeDirectory(destDir)
   except Vim.Fault.FileAlreadyExists, e:
      if scorch:
         print "  Deleting existing", destDir
         ds.CleanupDirectory(destDir)
         ds.MakeDirectory(destDir) # this better not fail..
      else:
         raise RuntimeError, "Directory %s already exists " + \
               "(add --scorch to delete it): %s" \
               % (destDir, str(e))

   # Find the iso
   isoPath = FindMemTestIso()
   vmx = "%s.vmx" % vmname

   localdir = tempfile.mkdtemp(prefix="hbrvmcfg.")

   rc = subprocess.call(["cp", isoPath, localdir])
   if rc != 0:
      raise RuntimeError, "Failed to copy %s to %s: %d" \
            % (isoPath, localdir, rc)

   CreateVMXConfigFile(vmx=os.path.join(localdir, vmx),
                       vmdk="unused",
                       nvram="unused",
                       displayName="%s" % vmname,
                       serialLog="vm-serial-output.txt",
                       floppyName="unused",
                       extraDisks=[],
                       comment="Diskless VM",
                       diskless=True,
                       isoName="memtest86-3.2.iso")

   # Copy everything up to the host
   CopyFiles(localdir,
             priHostd.Hostname(),
             datastore,
             destDir,
             priHostd.Username(),
             priHostd.Password())

   # Clean up the local copy of the VM
   shutil.rmtree(localdir)

   print "Created virtual machine", vmx

   # Register the VM on the current hostd
   cfgPath = ds.DatastorePath(os.path.join(destDir, vmx))
   try:
      tsk = priHostd.RegisterVm(cfg=cfgPath, name=vmname)
      priHostd.WaitForTask(tsk)
   except Vim.Fault.AlreadyExists, e:
      print "(VM %s (%s) already registered: %s)" % (vmname, cfgPath, str(e))

   vm = priHostd.FindVmByName(vmname)
   if not vm:
      raise RuntimeError, "Cannot find newly created vm %s" % vmname

   return priHostd.GetPyVimVM(vm)

def MungeConfigFile(cfgFile, disks, dropMacAddr=False):
   """
   Take an iterable cfgFile, and munge it for recovery using the given
   disks.

   The disks parameter is a dict mapping RDID to vmdk replica disk name.

   Returns an array of lines representing the munged config file.
   """
   oldDisks = {} # mapping from "scsiX:Y" to RDID
   oldLines = {} # mapping from "scsiX:Y" to config file line number

   result=[]
   for line in cfgFile:
      # Parse dictionary-format lines:
      #    <ws> <name> <ws> = <ws> <value> <ws> <comment>
      # OR
      #    <ws> <name> <ws> = <ws> " <quoted-value> " <ws> <comment>
      line = line.rstrip()
      comment = False
      m = re.match('^\s*([^= ]*)\s*=\s*"([^"]*)"\s*(#.*)?$', line)
      if not m:
         # Not a quoted value, maybe a bare value
         m = re.match('^\s*([^= ]*)\s*=\s*([^"#]*)\s*(#.*)?$', line)
         if not m:
            # Huh ... must be a comment or blank line?
            comment = True
            m = re.match('^\s*(#.*)?$', line)
            if not m:
               # Not a config file?
               raise RuntimeError, "Failed to parse config file line: %s" % line

      # Now that line is parsed, figure out if we want to drop or not
      drop=False
      if not comment:
         (name, value) = m.group(1,2)

         # Parse per-device HBR options.  Save scsiDev to RDID mapping.
         m = re.match('(scsi\d+:\d+).hbr_filter.rdid', name)
         if m:
            disk = m.group(1) # The 'scsiX:Y' string
            if not value:
               raise RuntimeError, "Scsi device %s does not have 'rdid' option?! (%s)" % (disk, value)
            oldDisks[disk] = value
            drop = True

         # Scrub the scsiDev ".fileName" key (replaced below) and track
         # this line number to make the munged file nicer.
         if not drop:
            m = re.match('(scsi\d+:\d+).fileName', name)
            if m:
               disk = m.group(1) # The 'scsiX:Y' string
               oldLines[disk] = len(result)
               result.append("XXX replaced by %s real fileName" % disk)
               drop = True

         # Strip the filter list (scsiX:Y.filters) entirely.  HBR is the
         # only filter that matters.
         #
         # Strip a couple other random keys, and anything that starts with
         # "hbr_filter.".
         #
         # Strip 'uuid.location' otherwise VM hangs at power-on with "Did
         # you copy me?"
         if not drop:
            drop = name.startswith("hbr_filter.") or \
                   name == "sched.swap.derivedName" or \
                   name == "replay.filename" or \
                   name == "uuid.location" or \
                   re.match('scsi\d+:\d+.filters', name) or \
                   re.match('^(ide|scsi)[0-9]+:[0-9]+\.hbr_filter\..*$', name)

         # Strip 'ethernet0.generatedAddress' and 'uuid.bios' if requested,
         # otherwise VM will power-on with same IP as primary VM.
         if not drop and dropMacAddr:
            drop = name == "uuid.bios" or \
                   name == "ethernet0.generatedAddress"

      if not drop:
         result.append(line)
      else:
         if False: #verbose: XXX
            print "Drop line:", line

   # Fixup the scsiX:Y.fileName lines that should point to replica disks
   # (Can't rely on the config file being in order, so do this after
   # parsing everything.)
   for scsiDev in sorted(oldDisks.iterkeys()):
      rdid = oldDisks[scsiDev]
      line = '%s.fileName = "%s"' % (scsiDev, disks[rdid])
      if False: #verbose: XXX
         print "Add line: ", line

      # Put the line at the right spot if we can
      if oldLines.has_key(scsiDev):
         result[oldLines[scsiDev]] = line
      else:
         result.append(line)

   result.append("# Automunged for HBR recovery on {0}".format(datetime.datetime.today()))
   return result

def CreateProtectedVM(ovfuri,
                      priHostd,
                      vmname,
                      datastore,
                      stdchurn=False,
                      churnrate=None,
                      disks=[],
                      cpfiles=[],
                      script=None,
                      stdblast=False,
                      baseDiskType=Vim.VirtualDiskManager.VirtualDiskType.thin,
                      snapDepth=0,
                      skipDisks=[]):
   """
   Common routine for creating a 'churn' VM and configuring it to run a
   simple startup script.

   @param[in] priHostd primary hostd connection object
   @param[in] vmname The name of the VM (used for displayname and vmx config file name, too)
   @param[in] datastore The name of the datastore to put the VM's files on (in priHostd)
   @param[in] scorch A bool indicating if any existing destDir should be removed
   @param[in] stdchurn A bool indicating if a "standard" churn configuration should be generated
   @param[in] churnrate A string "KB/min[,KBkeep][,RandomY/N][,Unmap %][,Compress %]" describing the argument to diskchurn2.sh
   @param[in] disks An (optional) list of disks (given by a string size in GB, can be fractional) to attach to VM
   @param[in] cpfiles An (optional) list of arbitrary files to copy into a fixed directory in the VM
   @param[in] script An (optional) script to run
   @param[in] stdblast A bool indicating if a "standard" disk blast configuration should be generated
   @param[in] baseDiskType The disk type to use when creating the base disk.
   @param[in] snapDepth Make each disk a child disk this number of snapshots deep.

   Optional parameters may be none.

   The disks specified are in addition to the 10MB base disk created for the VM.
   """

   deployment = ovflib.OVFManagerDeployment(ovfuri, priHostd)
   deployment.SetDatastore(datastore)
   deployment.SetName(vmname)

   vm = priHostd.GetPyVimVM(deployment.Deploy(powerOn=False))
   vmPathName = ParseDatastorePath(vm.GetConfig().GetFiles().GetVmPathName())
   destDir = os.path.dirname(vmPathName[1])

   priDs = priHostd.RemoteStorage(datastore)

   if not disks:
      disks = []

   if stdchurn or stdblast:
      # Add the parameters (if necesssary) to make a simple churn VM
      # A 1GB disk, the 'diskchurn.sh' script, and a command line that
      # runs diskchurn
      if not disks:
         disks = [ "1", ] # one GB disk

   if stdchurn and stdblast:
      raise RuntimeError, "Can't do both churn and blast at the same time."

   devLabels = DevLabelsFor(disks, skipDisks)

   if stdchurn or stdblast:

      if stdchurn:
         cpfiles = cpfiles + [ FindChurnScript() ] + FindChurnExtras()
      else:
         cpfiles = cpfiles + [ FindBlastScript() ] + FindBlastExtras()

      if not script:
         if not churnrate:
            churnrate = ""

         churnInArgs=churnrate.split(",")
         churnOutArgs=""

         if stdchurn:
            #
            # Parse the churn arguments.
            #
            if len(churnInArgs) >= 1 and churnInArgs[0]:
               churnOutArgs="%s -c %s" % (churnOutArgs, churnInArgs[0])

            if len(churnInArgs) >= 2 and churnInArgs[1]:
               churnOutArgs="%s -s %s" % (churnOutArgs, churnInArgs[1])

            if len(churnInArgs) >= 3 and churnInArgs[2]:
               if churnInArgs[2] == "y" or churnInArgs[2] == "Y":
                  churnOutArgs="%s -r" % churnOutArgs

            if len(churnInArgs) >= 4 and churnInArgs[3]:
               churnOutArgs="%s -u %s" % (churnOutArgs, churnInArgs[3])

            if len(churnInArgs) >= 5 and churnInArgs[4]:
               churnOutArgs="%s -k %s" % (churnOutArgs, churnInArgs[4])

            if len(churnInArgs) >= 6 and churnInArgs[5]:
               churnOutArgs="%s -w %s" % (churnOutArgs, churnInArgs[5])

            if len(churnInArgs) >= 7 and churnInArgs[6]:
               churnOutArgs="%s -y %s" % (churnOutArgs, churnInArgs[6])

            script = 'echo "Automated Disk Churnery:"\n' + \
                     'export PATH=".:${PATH}"\n' + \
                     'for disk in %s; do\n' % ' '.join(devLabels) + \
                     '  usleep 100\n' + \
                     '  diskchurn2.sh %s /dev/sd${disk} &\n' % (churnOutArgs) + \
                     'done\n' + \
                     'sleep 1\n' + \
                     'wait\n'
         else:
            #
            # Parse the blast arguments
            #
            if len(churnInArgs) >= 1 and churnInArgs[0]:
               if churnInArgs[0] == "y" or churnInArgs[0] == "Y":
                  churnOutArgs="%s -r" % churnOutArgs

            if len(churnInArgs) >= 2 and churnInArgs[1]:
               churnOutArgs="%s -u %s" % (churnOutArgs, churnInArgs[1])

            if len(churnInArgs) >= 3 and churnInArgs[2]:
               churnOutArgs="%s -k %s" % (churnOutArgs, churnInArgs[2])

            script = 'echo "Automated Disk Blastery:"\n' + \
                     'export PATH=".:${PATH}"\n' + \
                     'for disk in %s; do\n' % ' '.join(devLabels) + \
                     '  usleep 100\n' + \
                     '  diskblast.sh %s /dev/sd${disk} &\n' % (churnOutArgs) + \
                     'done\n' + \
                     'sleep 1\n' + \
                     'wait\n'

   cspec = Vim.Vm.ConfigSpec()
   envBrowser = priHostd.GetHostComputeResources().GetEnvironmentBrowser()
   cfgOption = envBrowser.QueryConfigOption(None, None)
   cfgTarget = envBrowser.QueryConfigTarget(None)

   cspec = pyVim.vmconfig.AddScsiCtlr(cspec, cfgOption=cfgOption, cfgTarget=cfgTarget,
                                      ctlrType='lsilogic')

   for disk in disks:
      capacityInKB = long(float(disk) * 1024 * 1024)
      capacityInB = long(capacityInKB * 1024)
      cspec = pyVim.vmconfig.AddDisk(cspec,
                                     cfgOption=cfgOption,
                                     cfgTarget=cfgTarget,
                                     thin=True,
                                     capacity=capacityInKB,
                                     capacityInBytes=capacityInB,
                                     datastorename=datastore)


   floppy = "%s.floppy" % vmname

   cspec = pyVim.vmconfig.AddFloppy(cspec,
                                    cfgOption=cfgOption,
                                    cfgTarget=cfgTarget,
                                    type='image',
                                    backingName=priDs.DatastorePath(
                                       '{}/{}'.format(destDir, floppy)))

   localdir = tempfile.mkdtemp(prefix="hbrvmcfg.")

   # Create the ttylinux boot floppy image
   cmd = CreateTestVmFloppy(localdir, os.path.join(localdir, floppy),
                            script, cpfiles, devLabels)
   # Copy everything up to the host
   CopyFiles(localdir,
             priHostd.Hostname(),
             datastore,
             destDir,
             priHostd.Username(),
             priHostd.Password())

   vm.Reconfigure(cspec)

   # Clean up the local copy of the VM
   shutil.rmtree(localdir)

   return vm
