PERFORCE change 125330 for review
Ivan Voras
ivoras at FreeBSD.org
Sat Aug 18 20:09:11 PDT 2007
http://perforce.freebsd.org/chv.cgi?CH=125330
Change 125330 by ivoras at ivoras_finstall on 2007/08/19 03:08:11
Finished framework for asynchronous backend jobs
Finished backend partitioning and file system creation primitives
Affected files ...
.. //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#11 edit
.. //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#2 edit
.. //depot/projects/soc2007/ivoras_finstall/installer/text/ninstall.txt#1 add
.. //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#6 edit
.. //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#9 edit
Differences ...
==== //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#11 (text+ko) ====
@@ -26,7 +26,7 @@
import logging
from types import MethodType
from xmlrpclib import ServerProxy
-import gtk, gtk.gdk, gtk.glade
+import gobject, gtk, gtk.gdk, gtk.glade
from basewin import BaseWin
from helpdialog import HelpDialog
@@ -45,7 +45,7 @@
{ "tile" : "nparts" },
{ "tile" : "ndefaultfs" },
{ "tile" : "nverify" },
- { "tile" : "ninstall" }
+ { "tile" : "ninstall", "glade" : "installprogress.glade" }
]
@@ -79,8 +79,12 @@
def _load_tile(self, tile_name):
"""Loads a tile by it's name and integrates it in the wizard window"""
+ glade_name = "glade/%s.glade" % tile_name
+ for t in self.step_track:
+ if t["tile"] == tile_name and "glade" in t:
+ glade_name = "glade/%s" % t["glade"]
self._clear_container(self.xml.get_widget("vbox_container"))
- self.tile_xml = gtk.glade.XML("glade/%s.glade" % tile_name)
+ self.tile_xml = gtk.glade.XML(glade_name)
self.tile_handlers = self._get_event_handlers(tile_name)
self.tile_xml.signal_autoconnect(self.tile_handlers)
w = self.tile_xml.get_widget("vbox_container").get_children()[0]
@@ -284,7 +288,11 @@
part_list = parts.keys()
part_list.sort()
for part in part_list:
- list.append([part, parts[part]["fs_type"], "%d MB" % parts[part]["mediasize"]])
+ if parts[part]["mediasize"] < 2048:
+ valid_target = "NO"
+ else:
+ valid_target = "Yes"
+ list.append([part, parts[part]["fs_type"], "%d MB" % parts[part]["mediasize"], valid_target])
self["parttree"].set_model(list)
self["parttree"].append_column(self._make_column("Partition", 0, True))
self["parttree"].append_column(self._make_column("File system", 1))
@@ -371,7 +379,8 @@
"size" : 512,
"name" : "%ss1" % self.trackdata["drive"],
"mount" : "/",
- "fs" : "UFS+SU"
+ "fs" : "UFS+SU",
+ "flags" : "init,active,boot"
})
parts.append({
"base" : self.trackdata["drive"],
@@ -395,7 +404,8 @@
"size" : var_size,
"name" : "%sa" % parts[2]["name"],
"mount" : "/var",
- "fs" : var_fs
+ "fs" : var_fs,
+ "flags" : "init"
})
parts.append({
"base" : parts[2]["name"],
@@ -408,7 +418,7 @@
parts.append({
"base" : parts[2]["name"],
"type" : "bsdlabel",
- "size" : parts[2]["size"] - parts[3]["size"] - parts[4]["size"],
+ "size" : parts[2]["size"] - parts[3]["size"] - parts[4]["size"] -2, # TODO: why is the bsdlabel 2679 sectors smaller than it should be?
"name" : "%sd" % parts[2]["name"],
"mount" : "/home",
"fs" : fs
@@ -425,7 +435,8 @@
"size" : 512,
"name" : "%ss1" % self.trackdata["drive"],
"mount" : "/",
- "fs" : "UFS+SU"
+ "fs" : "UFS+SU",
+ "flags" : "init,active,boot"
})
parts.append({
"base" : self.trackdata["drive"],
@@ -495,6 +506,37 @@
return True
+ def nverify_on_next(self):
+ self["button_cancel"].set_sensitive(False)
+ self["button_previous"].set_sensitive(False)
+ self["button_next"].set_sensitive(False)
+ return True
+
+
+ def ninstall_on_load(self):
+ self._load_label(self["label2"], "ninstall.txt")
+ self.trackdata["install_list"] = "Creating file systems...\n"
+ self._set_label(self["label3"], self.trackdata["install_list"])
+ self.trackdata["part_job"] = self.server.StartPartitionJob(self.trackdata["new_parts"])
+ gobject.timeout_add(500, self.part_progress)
+
+
+ def part_progress(self):
+ try:
+ pcnt = self.server.QueryJobProgress(self.trackdata["part_job"])
+ except Exception, e:
+ print "Exception ==================================================================="
+ print e
+ code, result = self.server.QueryJobError(self.trackdata["part_job"])
+ print code, result
+ return False
+ self["progressbar"].set_fraction(float(pcnt) / 100)
+ if pcnt == 100:
+ result = self.server.QueryJobResult(self.trackdata["part_job"])
+ print result
+ return False
+ return True
+
my_dir = os.path.split(sys.argv[0])[0]
==== //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#2 (text+ko) ====
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE glade-interface SYSTEM "glade-2.0.dtd">
-<!--Generated with glade3 3.2.2 on Fri Aug 17 01:01:37 2007 by ivoras at finstall.cosmos-->
+<!--Generated with glade3 3.2.2 on Sat Aug 18 15:28:34 2007 by ivoras at finstall.cosmos-->
<glade-interface>
<widget class="GtkWindow" id="window1">
<property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property>
@@ -36,8 +36,12 @@
<widget class="GtkLabel" id="label2">
<property name="visible">True</property>
<property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property>
+ <property name="xalign">0</property>
<property name="label" translatable="yes">label</property>
</widget>
+ <packing>
+ <property name="expand">False</property>
+ </packing>
</child>
<child>
<widget class="GtkVBox" id="vbox3">
@@ -47,11 +51,14 @@
<widget class="GtkLabel" id="label3">
<property name="visible">True</property>
<property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property>
+ <property name="xalign">0</property>
<property name="label" translatable="yes">label</property>
+ <property name="use_markup">True</property>
+ <property name="wrap">True</property>
</widget>
</child>
<child>
- <widget class="GtkProgressBar" id="progressbar1">
+ <widget class="GtkProgressBar" id="progressbar">
<property name="visible">True</property>
<property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property>
<property name="text" translatable="yes"></property>
==== //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#6 (text+ko) ====
@@ -21,19 +21,33 @@
# Interface to (most) FreeBSD's low-level utilities
-import os, sys
+import os, sys, popen2
+import logging
import re
import xmldict
cmd_sysctl = "/sbin/sysctl"
cmd_geom = "/sbin/geom"
cmd_mount = "/sbin/mount"
+cmd_fdisk = "/sbin/fdisk"
+cmd_bsdlabel = "/sbin/bsdlabel"
cmd_file = "/usr/bin/file -s"
+cmd_newfs = "/sbin/newfs"
+cmd_gjournal = "/sbin/gjournal"
+cmd_kldstat = "/sbin/kldstat"
+cmd_kldload = "/sbin/kldload"
+cmd_mke2fs = "/usr/local/sbin/mke2fs"
file_dmesg = "/var/run/dmesg.boot"
+bsdlabel_map = ('a', 'b', 'd', 'e', 'f', 'g', 'h') # maps bsdlabel # to a letter
+
def get_sysctl(name, b_flag = False):
+ """
+ Returns a sysctl value. If second parameter is True, the sysctl is treated
+ as binary buffer (or a string)
+ """
global cmd_sysctl
if b_flag:
str = os.popen("%s -b %s" % (cmd_sysctl, name)).read().strip()
@@ -98,10 +112,181 @@
return m.group(1)
+def make_temp_script():
+ """Creates a temporary file to hold scripts for various utilities"""
+ fname = os.tempnam()
+ f = file(fname, "w+")
+ return (fname, f)
+
+
+def tolist(e):
+ if type(e) == type([]):
+ return e
+ else:
+ return [e]
+
+
+def geom_sector_size(dev):
+ xml = get_geom_xml()
+ for cls in xml["mesh"]["class"]:
+ if "geom" in cls:
+ for geom in tolist(cls["geom"]):
+ if "provider" in geom:
+ for provider in tolist(geom["provider"]):
+ if provider["name"].data == dev and "sectorsize" in provider:
+ return int(provider["sectorsize"].data)
+ return None
+
+
+def exec_cmd(cmd, input = None):
+ """
+ Convenience function that executes the command specified by
+ the cmd argument and returns its exit status and its output from both stdout
+ and stderr. Optionally, a simple string can be send to the process'
+ stdin.
+ """
+ logging.info("Executing %s" %cmd)
+ p = popen2.Popen4(cmd)
+ if input != None:
+ p.tochild.write(input)
+ p.tochild.flush()
+ buf = ""
+ while p.poll() == -1:
+ buf += p.fromchild.read(4096)
+ buf += p.fromchild.read()
+ p.fromchild.close()
+ p.tochild.close()
+ w = p.wait()
+ return (os.WEXITSTATUS(w), buf)
+
+
+def create_fdisk_partition(dev, index, offset_sectors, size_sectors, flags=[]):
+ global cmd_fdisk
+ temp_fname, f = make_temp_script()
+ # Create the script
+ f.write("p %d 165 %d %d\n" % (index, offset_sectors, size_sectors))
+ if "active" in flags:
+ f.write("a %d\n" % index)
+ f.close()
+ # Execute the script
+ if "init" in flags:
+ cmd = "%s -i -f %s %s" % (cmd_fdisk, temp_fname, dev)
+ else:
+ cmd = "%s -f %s %s" % (cmd_fdisk, temp_fname, dev)
+ code, output = exec_cmd(cmd)
+ os.unlink(temp_fname)
+ if code != 0:
+ return (code, output, None)
+ if 'boot' in flags:
+ # Make the device bootable
+ code, o = exec_cmd("%s -B %s" % (cmd_fdisk, dev), "y\ny\n")
+ output += o
+ if code != 0:
+ return (code, o, None)
+ # Make fdisk report what it has done
+ cmd = "%s -p %s" % (cmd_fdisk, dev)
+ code, dump = exec_cmd(cmd)
+ if code != 0:
+ return (code, None, None)
+ m = re.search(r"p %d .+ (\d+) (\d+)" % index, dump)
+ if m == None:
+ return (1, None, None)
+ p_start = int(m.group(1))
+ p_length = int(m.group(2))
+ return (0, output, p_start+p_length)
+
+
+def create_bsdlabel_partition(dev, index, offset_sectors, size_sectors, flags=[], fs_type="4.2BSD"):
+ global cmd_bsdlabel, bsdlabel_map
+ output = ""
+ if "init" in flags:
+ # Initialize the bsdlabel table
+ code, o = exec_cmd("%s -w %s" % (cmd_bsdlabel, dev))
+ output += o
+ if code != 0:
+ return (code, output, None)
+ if "boot" in flags:
+ # Install boot loader
+ code, o = exec_cmd("%s -B %s" % (cmd_bsdlabel, dev))
+ output += o
+ if code != 0:
+ return (code, output, None)
+ # Read the current label
+ code, label = exec_cmd("%s %s" % (cmd_bsdlabel, dev))
+ if code != 0:
+ return (code, label, None)
+ label = label.replace("\t", " ") # get rid of multiple whitespace
+ while label.find(" ") != -1:
+ label = label.replace(" ", " ")
+ label_found = False
+ lines = [x.strip() for x in label.split("\n")]
+ for i, line in enumerate(lines):
+ if line.find(":") == -1:
+ continue
+ letter, rest = [x.strip() for x in line.split(":", 1)]
+ rest = [x.strip() for x in rest.split(" ")]
+ if bsdlabel_map[index] == letter: # modify the label "in place"
+ label_found = True
+ lines[i] = "%s: %d %d %s" % (letter, size_sectors, offset_sectors, fs_type)
+ if not label_found:
+ lines.append("%s: %d %d %s" % (bsdlabel_map[index], size_sectors, offset_sectors, fs_type))
+ # Create the bsdlabel script and run it through bsdlabel
+ script_fname, f = make_temp_script()
+ f.write("\n".join(lines))
+ f.write("\n")
+ f.close()
+ code, output = exec_cmd("%s -R %s %s" % (cmd_bsdlabel, dev, script_fname))
+ #os.unlink(script_fname)
+ if code != 0:
+ return (code, output, None)
+ # Verify the result & fetch the last_offset
+ code, label = exec_cmd("%s %s" % (cmd_bsdlabel, dev))
+ if code != 0:
+ return (code, label, None)
+ m = re.search(r"\s+%s:\s+(\d+)\s+(\d+)" % bsdlabel_map[index], label)
+ if m == None:
+ return (1, "Cannot parse the label:\n"+label, None)
+ p_length = int(m.group(1))
+ p_offset = int(m.group(2))
+ return (0, output, p_offset+p_length)
+
+
+def newfs_simple(dev, fs_type):
+ """Creates a "simple" file system on the given device"""
+ global cmd_newfs, cmd_mke2fs
+ if not dev.startswith("/dev/"):
+ dev = "/dev/"+dev
+ if fs_type == "UFS":
+ cmd = "%s %s" % (cmd_newfs, dev)
+ elif fs_type == "UFS+SU":
+ cmd = "%s -U %s" % (cmd_newfs, dev)
+ elif fs_type == "Ext2":
+ cmd = "%s %s" % (cmd_mke2fs, dev)
+ else:
+ return (1, "Unknown file system type "+fs_type)
+ return exec_cmd(cmd)
+
+
+def newfs_ufsgj(dev):
+ """Create a gjournaled UFS on the given device"""
+ global cmd_newfs, cmd_gjournal, cmd_kldstat, cmd_kldload
+ if not dev.startswith("/dev/"):
+ dev = "/dev/"+dev
+ code, kldlist = exec_cmd(cmd_kldstat)
+ if kldlist.find("geom_journal") == -1:
+ code, output = exec_cmd("%s geom_journal" % cmd_kldload)
+ if code != 0:
+ return (code, "Cannot kldload geom_journal: "+output)
+ code, output = exec_cmd("%s label -f %s" % (cmd_gjournal, dev))
+ if code != 0:
+ return (code, "Canno label gjournal: "+output)
+ return exec_cmd("%s -J %s.journal" % (cmd_newfs, dev))
+
+
if __name__ == "__main__":
xml = get_geom_xml()
for cls in xml["mesh"]["class"]:
if cls["name"].data == "DISK":
for geom in cls["geom"]:
- print geom["name"].data, geom["provider"]["mediasize"].data
+ print geom["name"].data, geom["provider"]["mediasize"].data, geom_sector_size(geom["name"].data)
==== //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#9 (text+ko) ====
@@ -23,8 +23,9 @@
import os, sys
import re
-import logging
+import logging, warnings
from threading import Thread, Lock
+from StringIO import StringIO
import globals
import freebsd
@@ -52,6 +53,10 @@
return [e]
+class SysToolJobException(Exception):
+ pass
+
+
class SysToolJob(Thread):
"""A generic asynchronous SysTool job"""
@@ -68,16 +73,98 @@
class PartitionJob(SysToolJob):
"""A partitioning SysTool job. This one accept a list of partitions
- to create and creates them one by one."""
+ to create and creates them one by one. The list cannot be arbitrary,
+ it is evaluated sequentially. After creation, the partitions are also
+ formatted with the file system specified (if any).
+ Elements of part_spec list are dictionaries which contain keys:
+ base - base device on which to create the part.
+ type - partition type
+ size - size in MB
+ name - expected name of the created part.
+ fs - file system to create on the part.
+ type can be: fdisk | bsdlabel | zvol | zfs,
+ and fs can be: UFS+SU | UFS+GJ | Ext2 | ZFS
+ If error is encountered during any of the operations, the job is
+ terminated, self.finished, self.error and self.result are set.
+ """
def __init__(self, part_spec):
SysToolJob.__init__(self)
self.part_spec = part_spec
+
def run(self):
+ # A thread
+ buf = StringIO()
+ fdisk_part_nr = 1 # fdisk partitions start from 1
+ fdisk_last_offset = 1 # in sectors
+ bsdlabel_nr = 0 # bsdlabels start from 0
+ bsdlabel_last_offset = 1
for i, part in enumerate(self.part_spec):
self.percent_complete = self._calc_percent(i, len(self.part_spec))
+ if "flags" in part:
+ flags = [x.strip() for x in part["flags"].split(",")]
+ else:
+ flags = []
+ if part["type"] == "fdisk":
+ # Create a fdisk partition
+ if "init" in flags:
+ fdisk_part_nr = 1
+ fdisk_last_offset = 1
+ if fdisk_part_nr > 4:
+ self.error = 2
+ self.result = "Too many fdisk partitions"
+ break
+ sector_size = freebsd.geom_sector_size(part["base"])
+ sectors_per_mb = (1024*1024) / sector_size
+ part_size_sectors = sectors_per_mb * part["size"]
+ code, result, last_offset = freebsd.create_fdisk_partition(part["base"], fdisk_part_nr, fdisk_last_offset, part_size_sectors, flags)
+ buf.write(result)
+ if code != 0: # error
+ self.error = code
+ self.result = buf.getvalue()
+ break # Bail out on first error
+ fdisk_last_offset = last_offset
+ fdisk_part_nr += 1
+ elif part["type"] == "bsdlabel":
+ # Create a bsdlabel partition
+ if "init" in flags:
+ bsdlabel_nr = 0
+ bsdlabel_last_offset = 1
+ if bsdlabel_nr > 7:
+ self.error = 2
+ self.result = "Too many bsdlabel partitions"
+ break
+ sector_size = freebsd.geom_sector_size(part["base"])
+ sectors_per_mb = (1024*1024) / sector_size
+ part_size_sector = sectors_per_mb * part["size"]
+ code, result, last_offset = freebsd.create_bsdlabel_partition(part["base"], bsdlabel_nr, bsdlabel_last_offset, part_size_sector, flags)
+ buf.write(result)
+ if code != 0:
+ self.error = code
+ self.result = buf.getvalue()
+ break
+ bsdlabel_last_offset = last_offset
+ bsdlabel_nr += 1
+ elif part["type"] == "zpool":
+ pass
+ if part["fs"] in ("UFS", "UFS+SU", "Ext2"):
+ code, result = freebsd.newfs_simple(part["name"], part["fs"])
+ buf.write(result)
+ if code != 0:
+ self.error = code
+ break
+ elif part["fs"] == "UFS+GJ":
+ code, result = freebsd.newfs_ufsgj(part["name"])
+ buf.write(result)
+ if code != 0:
+ self.error = code
+ break
+ if self.result == None:
+ self.result = buf.getvalue()
+ self.finished = True
+
class SysToolEngine:
def __init__(self):
@@ -85,6 +172,7 @@
self.root_live = "" # Live file system root (for binaries!)
self.job_list = []
self.job_list_lock = Lock()
+ warnings.simplefilter('ignore', RuntimeWarning)
def GetId(self):
@@ -281,8 +369,10 @@
@logexception
def StartPartitionJob(self, part_spec):
- """Starts a job that executes a partition spec list. Returns
- an integer job_id"""
+ """
+ Starts a job that executes a partition spec list. Returns
+ an integer job_id
+ """
self.job_list_lock.acquire()
job = PartitionJob(part_spec)
self.job_list.append(job)
@@ -294,21 +384,28 @@
@logexception
def QueryJobProgress(self, job_id):
- """Queries the progress of a job, returns percent complete or None
- if the job is in error"""
+ """
+ Queries the progress of a job, returns percent complete or None
+ if the job is in error
+ """
self.job_list_lock.acquire()
job = self.job_list[job_id-1]
self.job_list_lock.release()
if job.error != None:
- return None
- return job.percent_complete
+ raise SysToolJobException, "Error %d, %s" % (job.error, job.result)
+ if not job.finished:
+ return job.percent_complete
+ else:
+ return 100
@logexception
def QueryJobResult(self, job_id):
- """Queries the result of a job, if the job is finished. Returns
+ """
+ Queries the result of a job, if the job is finished. Returns
a string with the job's status report or None if the job is not
- yet finished."""
+ yet finished.
+ """
self.job_list_lock.acquire()
job = self.job_list[job_id-1]
self.job_list_lock.release()
@@ -316,3 +413,14 @@
return None
return job.result
+
+ @logexception
+ def QueryJobError(self, job_id):
+ """Queries job error status."""
+ self.job_list_lock.acquire()
+ job = self.job_list[job_id-1]
+ self.job_list_lock.release()
+ if job.error == None:
+ return None
+ return (job.error, job.result)
+
More information about the p4-projects
mailing list