Util: Replace mkblankimage.sh with the new gem5img.py.

This change replaces the mkblankimage.sh script, used for creating new disk
images, with a new gem5img.py script. The new version is written in python
instead of bash, takes its parameters from command line arguments instead of
prompting for them, and finds a free loopback device dynamically instead of
hardcoding /dev/loop1. The file system used is now optionally configurable,
and the blank image is filled by a "hole" left by lseek and write instead of
literally filling it with zeroes.

The functionality of the new script is broken into subcommands "init",
"mount", "umount", "new", "partition", and "format". "init" creates a new file
of the appropriate size, partitions it, and then formats the first (and only)
new parition. "mount" attaches a new loopback device to the first parition of
the image file and mounts it to the specified mount point. "umount" unmounts
the specified mount point and identifies and cleans up the underlying loopback
device. "new", "partition", and "format" are the individual stages of "init"
but broken out so they can be run individually. That's so an image can be
reinitialized in place if needed.

Two features of the original script are being dropped. The first is the
ability to specify a source directory to copy into the new file system. The
second is the ability to specify a list of commands to run which are expected
to (but not required to) update the permissions of the files in the new fs.
Both of these seem easy enough to do manually, especially given the "mount"
and "umount" commands, that removing them would meaningfully simplify the
script without making it less useful.

diff --git a/util/gem5img.py b/util/gem5img.py
new file mode 100755 (executable)
index 0000000..f02ccb1
--- /dev/null
+++ b/util/gem5img.py
@@ -0,0 +1,372 @@
+#!/usr/bin/python
+#
+# gem5img.py
+# Script for managing a gem5 disk image.
+#
+
+from optparse import OptionParser
+import os
+from os import environ as env
+import string
+from subprocess import CalledProcessError, Popen, PIPE, STDOUT
+from sys import exit, argv
+
+
+# Some constants.
+MaxLBACylinders = 16383
+MaxLBAHeads = 16
+MaxLBASectors = 63
+MaxLBABlocks = MaxLBACylinders * MaxLBAHeads * MaxLBASectors
+
+BlockSize = 512
+MB = 1024 * 1024
+
+# Setup PATH to look in the sbins.
+env['PATH'] += ':/sbin:/usr/sbin'
+
+# Whether to print debug output.
+debug = False
+
+# Figure out cylinders, heads and sectors from a size in blocks.
+def chsFromSize(sizeInBlocks):
+    if sizeInBlocks >= MaxLBABlocks:
+        sizeInMBs = (sizeInBlocks * BlockSize) / MB
+        print '%d MB is too big for LBA, truncating file.' % sizeInMBs
+        return (MaxLBACylinders, MaxLBAHeads, MaxLBASectors)
+
+    sectors = sizeInBlocks
+    if sizeInBlocks > 63:
+        sectors = 63
+
+    headSize = sizeInBlocks / sectors
+    heads = 16
+    if headSize < 16:
+        heads = sizeInBlocks
+
+    cylinders = sizeInBlocks / (sectors * heads)
+
+    return (cylinders, heads, sectors)
+
+
+# Figure out if we should use sudo.
+def needSudo():
+    if not hasattr(needSudo, 'notRoot'):
+        needSudo.notRoot = (os.geteuid() != 0)
+        if needSudo.notRoot:
+            print 'You are not root. Using sudo.'
+    return needSudo.notRoot
+
+# Run an external command.
+def runCommand(command, inputVal=''):
+    print "%>", ' '.join(command)
+    proc = Popen(command, stdin=PIPE)
+    proc.communicate(inputVal)
+    return proc.returncode
+
+# Run an external command and capture its output. This is intended to be
+# used with non-interactive commands where the output is for internal use.
+def getOutput(command, inputVal=''):
+    global debug
+    if debug:
+        print "%>", ' '.join(command)
+    proc = Popen(command, stderr=STDOUT,
+                 stdin=PIPE, stdout=PIPE)
+    (out, err) = proc.communicate(inputVal)
+    return (out, proc.returncode)
+
+# Run a command as root, using sudo if necessary.
+def runPriv(command, inputVal=''):
+    realCommand = command
+    if needSudo():
+        realCommand = [findProg('sudo')] + command
+    return runCommand(realCommand, inputVal)
+
+def privOutput(command, inputVal=''):
+    realCommand = command
+    if needSudo():
+        realCommand = [findProg('sudo')] + command
+    return getOutput(realCommand, inputVal)
+
+# Find the path to a program.
+def findProg(program, cleanupDev=None):
+    (out, returncode) = getOutput(['which', program])
+    if returncode != 0:
+        if cleanupDev:
+            cleanupDev.destroy()
+        exit("Unable to find program %s, check your PATH variable." % program)
+    return string.strip(out)
+
+class LoopbackDevice(object):
+    def __init__(self, devFile=None):
+        self.devFile = devFile
+    def __str__(self):
+        return str(self.devFile)
+
+    def setup(self, fileName, offset=False):
+        assert not self.devFile
+        (out, returncode) = privOutput([findProg('losetup'), '-f'])
+        if returncode != 0:
+            print out
+            return returncode
+        self.devFile = string.strip(out)
+        command = [findProg('losetup'), self.devFile, fileName]
+        if offset:
+            off = findPartOffset(self.devFile, fileName, 0)
+            command = command[:1] + \
+                      ["-o", "%d" % off] + \
+                      command[1:]
+        return runPriv(command)
+
+    def destroy(self):
+        assert self.devFile
+        returncode = runPriv([findProg('losetup'), '-d', self.devFile])
+        self.devFile = None
+        return returncode
+
+def findPartOffset(devFile, fileName, partition):
+    # Attach a loopback device to the file so we can use sfdisk on it.
+    dev = LoopbackDevice()
+    dev.setup(fileName)
+    # Dump the partition information.
+    command = [findProg('sfdisk'), '-d', dev.devFile]
+    (out, returncode) = privOutput(command)
+    if returncode != 0:
+        print out
+        exit(returncode)
+    lines = out.splitlines()
+    # Make sure the first few lines of the output look like what we expect.
+    assert(lines[0][0] == '#')
+    assert(lines[1] == 'unit: sectors')
+    assert(lines[2] == '')
+    # This line has information about the first partition.
+    chunks = lines[3].split()
+    # The fourth chunk is the offset of the partition in sectors followed by
+    # a comma. We drop the comma and convert that to an integer.
+    sectors = string.atoi(chunks[3][:-1])
+    # Free the loopback device and return an answer.
+    dev.destroy()
+    return sectors * BlockSize
+
+def mountPointToDev(mountPoint):
+    (mountTable, returncode) = getOutput([findProg('mount')])
+    if returncode != 0:
+        print mountTable
+        exit(returncode)
+    mountTable = mountTable.splitlines()
+    for line in mountTable:
+        chunks = line.split()
+        if os.path.samefile(chunks[2], mountPoint):
+            return LoopbackDevice(chunks[0])
+    return None
+
+
+# Commands for the gem5img.py script
+commands = {}
+commandOrder = []
+
+class Command(object):
+    def addOption(self, *args, **kargs):
+        self.parser.add_option(*args, **kargs)
+
+    def __init__(self, name, description, posArgs):
+        self.name = name
+        self.description = description
+        self.func = None
+        self.posArgs = posArgs
+        commands[self.name] = self
+        commandOrder.append(self.name)
+        usage = 'usage: %prog [options]'
+        posUsage = ''
+        for posArg in posArgs:
+            (argName, argDesc) = posArg
+            usage += ' %s' % argName
+            posUsage += '\n  %s: %s' % posArg
+        usage += posUsage
+        self.parser = OptionParser(usage=usage, description=description)
+        self.addOption('-d', '--debug', dest='debug', action='store_true',
+                       help='Verbose output.')
+
+    def parseArgs(self, argv):
+        (self.options, self.args) = self.parser.parse_args(argv[2:])
+        if len(self.args) != len(self.posArgs):
+            self.parser.error('Incorrect number of arguments')
+        global debug
+        if self.options.debug:
+            debug = True
+
+    def runCom(self):
+        if not self.func:
+            exit('Unimplemented command %s!' % self.name)
+        self.func(self.options, self.args)
+
+
+# A command which prepares an image with an partition table and an empty file
+# system.
+initCom = Command('init', 'Create an image with an empty file system.',
+                  [('file', 'Name of the image file.'),
+                   ('mb', 'Size of the file in MB.')])
+initCom.addOption('-t', '--type', dest='fstype', action='store',
+                  default='ext2',
+                  help='Type of file system to use. Appended to mkfs.')
+
+# A command to mount the first partition in the image.
+mountCom = Command('mount', 'Mount the first partition in the disk image.',
+                   [('file', 'Name of the image file.'),
+                    ('mount point', 'Where to mount the image.')])
+
+def mountComFunc(options, args):
+    (path, mountPoint) = args
+    if not os.path.isdir(mountPoint):
+        print "Mount point %s is not a directory." % mountPoint
+
+    dev = LoopbackDevice()
+    if dev.setup(path, offset=True) != 0:
+        exit(1)
+
+    if runPriv([findProg('mount'), str(dev), mountPoint]) != 0:
+        dev.destroy()
+        exit(1)
+
+mountCom.func = mountComFunc
+
+# A command to unmount the first partition in the image.
+umountCom = Command('umount', 'Unmount the first partition in the disk image.',
+                    [('mount point', 'What mount point to unmount.')])
+
+def umountComFunc(options, args):
+    (mountPoint,) = args
+    if not os.path.isdir(mountPoint):
+        print "Mount point %s is not a directory." % mountPoint
+        exit(1)
+
+    dev = mountPointToDev(mountPoint)
+    if not dev:
+        print "Unable to find mount information for %s." % mountPoint
+
+    # Unmount the loopback device.
+    if runPriv([findProg('umount'), mountPoint]) != 0:
+        exit(1)
+
+    # Destroy the loopback device.
+    dev.destroy()
+
+umountCom.func = umountComFunc
+
+
+# A command to create an empty file to hold the image.
+newCom = Command('new', 'File creation part of "init".',
+                 [('file', 'Name of the image file.'),
+                  ('mb', 'Size of the file in MB.')])
+
+def newImage(file, mb):
+    (cylinders, heads, sectors) = chsFromSize((mb * MB) / BlockSize)
+    size = cylinders * heads * sectors * BlockSize
+
+    # We lseek to the end of the file and only write one byte there. This
+    # leaves a "hole" which many file systems are smart enough not to actually
+    # store to disk and which is defined to read as zero.
+    fd = os.open(file, os.O_WRONLY | os.O_CREAT)
+    os.lseek(fd, size - 1, os.SEEK_SET)
+    os.write(fd, '\0')
+
+def newComFunc(options, args):
+    (file, mb) = args
+    mb = string.atoi(mb)
+    newImage(file, mb)
+
+
+newCom.func = newComFunc
+
+# A command to partition the image file like a raw disk device.
+partitionCom = Command('partition', 'Partition part of "init".',
+                       [('file', 'Name of the image file.')])
+
+def partition(dev, cylinders, heads, sectors):
+    # Use fdisk to partition the device
+    comStr = '0,\n;\n;\n;\n'
+    return runPriv([findProg('sfdisk'), '--no-reread', '-D', \
+                   '-C', "%d" % cylinders, \
+                   '-H', "%d" % heads, \
+                   '-S', "%d" % sectors, \
+                   str(dev)], inputVal=comStr)
+
+def partitionComFunc(options, args):
+    (path,) = args
+
+    dev = LoopbackDevice()
+    if dev.setup(path) != 0:
+        exit(1)
+
+    # Figure out the dimensions of the file.
+    size = os.path.getsize(path)
+    if partition(dev, *chsFromSize(size / BlockSize)) != 0:
+        dev.destroy()
+        exit(1)
+
+    dev.destroy()
+
+partitionCom.func = partitionComFunc
+
+# A command to format the first partition in the image.
+formatCom = Command('format', 'Formatting part of "init".',
+                    [('file', 'Name of the image file.')])
+formatCom.addOption('-t', '--type', dest='fstype', action='store',
+                    default='ext2',
+                    help='Type of file system to use. Appended to mkfs.')
+
+def formatImage(dev, fsType):
+    return runPriv([findProg('mkfs.%s' % fsType, dev), str(dev)])
+
+def formatComFunc(options, args):
+    (path,) = args
+
+    dev = LoopbackDevice()
+    if dev.setup(path, offset=True) != 0:
+        exit(1)
+
+    # Format the device.
+    if formatImage(dev, options.fstype) != 0:
+        dev.destroy()
+        exit(1)
+
+    dev.destroy()
+
+formatCom.func = formatComFunc
+
+def initComFunc(options, args):
+    (path, mb) = args
+    mb = string.atoi(mb)
+    newImage(path, mb)
+    dev = LoopbackDevice()
+    if dev.setup(path) != 0:
+        exit(1)
+    size = os.path.getsize(path)
+    if partition(dev, *chsFromSize((mb * MB) / BlockSize)) != 0:
+        dev.destroy()
+        exit(1)
+    dev.destroy()
+    if dev.setup(path, offset=True) != 0:
+        exit(1)
+    if formatImage(dev, options.fstype) != 0:
+        dev.destroy()
+        exit(1)
+    dev.destroy()
+
+initCom.func = initComFunc
+
+
+# Figure out what command was requested and execute it.
+if len(argv) < 2 or argv[1] not in commands:
+    print 'Usage: %s [command] <command arguments>'
+    print 'where [command] is one of '
+    for name in commandOrder:
+        command = commands[name]
+        print '    %s: %s' % (command.name, command.description)
+    print 'Watch for orphaned loopback devices and delete them with'
+    print 'losetup -d. Mounted images will belong to root, so you may need'
+    print 'to use sudo to modify their contents.'
+    exit(1)
+
+command = commands[argv[1]]
+command.parseArgs(argv)
+command.runCom()

diff --git a/util/mkblankimage.sh b/util/mkblankimage.sh
deleted file mode 100755 (executable)
index e764ea4..0000000
--- a/util/mkblankimage.sh
+++ /dev/null
@@ -1,219 +0,0 @@
-#!/bin/bash
-#
-# makeblankimage.sh
-# Make a blank M5 disk image
-#
-
-while getopts "m" OPT
-do
-        case "$OPT" in
-                m)  MOUNT_IT=1
-        esac
-done
-
-DEBUG=0
-
-if [ $DEBUG -ne 0 ]; then
-    set -x -e
-    OUTPUT=""
-else
-    OUTPUT="> /dev/null 2>&1"
-fi
-
-abort() {
-    echo $@
-    exec /bin/false
-}
-
-find_prog() {
-    PROG_PATH=`which $1`
-    if [ $? -ne 0 ]; then
-        abort "Unable to find program $1, check your PATH variable"
-    fi
-    echo $PROG_PATH
-}
-
-run_priv() {
-    if [ "$HAVE_SUDO" = "y" ]; then
-        eval $SUDO $@ $OUTPUT
-    else
-        eval $@ $OUTPUT
-    fi
-
-    if [ $? -ne 0 ]; then
-        abort "Failed to run $@ as root"
-    fi
-}
-
-usage() {
-    abort "Usage: $0 [root-path to copy] [extra ownership commands ...]"
-}
-
-# Setup PATH to look in the sbins
-export PATH=$PATH:/sbin:/usr/sbin
-
-# Get all of the programs needed, or exit
-DD=`find_prog dd`
-SFDISK=`find_prog sfdisk`
-LOSETUP=`find_prog losetup`
-SUDO=`find_prog sudo`
-MKE2FS=`find_prog mke2fs`
-MKDIR=`find_prog mkdir`
-MOUNT=`find_prog mount`
-UMOUNT=`find_prog umount`
-WHOAMI=`find_prog whoami`
-CP=`find_prog cp`
-CHOWN=`find_prog chown`
-
-# Prompt for the root password, if needed
-CUR_USER=`$WHOAMI`
-
-if [ $# -ge 1 ]; then
-    if [ ! $MOUNT_IT ]; then
-        ROOT_PATH=$1
-
-        if [ ! -d $ROOT_PATH ]; then
-                usage
-        fi
-   else
-        ROOT_PATH=""
-   fi
-else
-    ROOT_PATH=""
-fi
-
-if [ ! "$CUR_USER" = "root" ]; then
-    echo -n "Do you have sudo access? [y/n] "
-    read HAVE_SUDO
-
-    if [ ! "$HAVE_SUDO" = "y" ]; then
-        abort "You must have sudo access or run this script as root"
-    fi
-fi
-
-echo -n "How large do you want this disk image (in MB): "
-read USER_SIZE_MB
-
-# size in bytes = SIZE_MB * 1024 * 1024
-# size in blocks = SIZE_BYTE / 512
-let BLK_SIZE=$USER_SIZE_MB*1024*2
-
-let MAX_LBA=16383*16*63
-
-if [ $BLK_SIZE -ge $MAX_LBA ]; then
-    CYLS=16383
-    HEADS=16
-    SECTORS=63
-else
-    # Set Sectors
-    if [ $BLK_SIZE -ge 63 ]; then
-        SECTORS=63
-    else
-        SECTORS=$BLK_SIZE
-    fi
-
-    # Set Heads
-    let HEAD_SIZE=$BLK_SIZE/$SECTORS
-
-    if [ $HEAD_SIZE -ge 16 ]; then
-        HEADS=16
-    else
-        HEADS=$BLK_SIZE
-    fi
-
-    # Set Cylinders
-    let SEC_HEAD=$SECTORS*$HEADS
-    let CYLS=$BLK_SIZE/$SEC_HEAD
-fi
-
-# Recalculate number of sectors
-let BLK_SIZE=$CYLS*$HEADS*$SECTORS
-
-# Get the name of the file and directory to build in
-echo -n "What directory would you like to build the image in? "
-read IMAGE_DIR
-
-if [ ! -d $IMAGE_DIR ]; then
-    abort "The directory $IMAGE_DIR does not exist"
-fi
-
-echo -n "What would you like to name the image? "
-read IMAGE_NAME
-
-IMAGE_FILE=$IMAGE_DIR/$IMAGE_NAME
-
-# DD the blank image
-echo
-echo "dd'ing the blank image (this make take a while)..."
-eval $DD if=/dev/zero of=$IMAGE_FILE bs=512 count=$BLK_SIZE $OUTPUT
-if [ $? -ne 0 ]; then
-    abort "Unable to create the blank image $IMAGE_NAME in $IMAGE_DIR"
-fi
-
-# losetup the image with no offset to do the fdisk
-echo
-echo "Binding the image and partitioning..."
-run_priv $LOSETUP /dev/loop0 $IMAGE_FILE
-if [ $? -ne 0 ]; then
-    abort "losetup to /dev/loop0 failed, make sure nothing is setup on loop0 (check by typing 'mount') "
-fi
-
-# fdisk the image
-run_priv $SFDISK --no-reread -D -C $CYLS -H $HEADS -S $SECTORS /dev/loop0 <<EOF
-0,
-;
-;
-;
-EOF
-
-# Un-losetup the image
-run_priv $LOSETUP -d /dev/loop0
-
-# Mount the image with an offset and make the filesystem
-echo
-echo "Remounting image and formatting..."
-let BASE_OFFSET=63*512
-
-run_priv $LOSETUP -o $BASE_OFFSET /dev/loop0 $IMAGE_FILE
-
-run_priv $MKE2FS /dev/loop0
-
-# If a root path was specified then copy the root path into the image
-if [ ! -z "$ROOT_PATH" ]; then
-    echo "Copying root from $ROOT_PATH to image file"
-
-    run_priv $MKDIR -p /tmp/mnt
-
-    run_priv $MOUNT /dev/loop0 /tmp/mnt
-
-    run_priv $CP -a $ROOT_PATH/* /tmp/mnt
-
-    run_priv $CHOWN -R root.root /tmp/mnt
-
-    # run extra permissions while disk is mounted
-    TOPDIR=`pwd`
-    cd /tmp/mnt
-    i=2
-    while [ $i -le $# ]; do
-        run_priv ${!i}
-        let i=i+1
-    done
-    cd $TOPDIR
-
-    run_priv $UMOUNT /tmp/mnt
-fi
-
-run_priv $LOSETUP -d /dev/loop0
-
-
-if [ $MOUNT_IT -eq 1 ]; then
-        run_priv mount -o loop,offset=$BASE_OFFSET $IMAGE_FILE /tmp/mnt
-else
-        echo
-        echo "Disk image creation complete."
-        echo "To mount the image, run the following commands:"
-        echo "# $MOUNT -o loop,offset=$BASE_OFFSET $IMAGE_FILE /mount/point"
-        echo
-        echo "And to unmount the image, run:"
-        echo "# $UMOUNT /mount/point"
-fi;