PERFORCE change 123518 for review

Garrett Cooper gcooper at FreeBSD.org
Sun Jul 15 04:45:26 UTC 2007


http://perforce.freebsd.org/chv.cgi?CH=123518

Change 123518 by gcooper at optimus-revised_pkgtools on 2007/07/15 04:45:24

	Summary:
		Performance (?) and functionality improvements on simple profiling scripts.
	
	run_prof -- Create timestamp instead of depending on other scripts to do so.
	
	prof_postprocess --
		-Improve error checking.
		-Get rid of use Data::Dumper.
		-Rename $hash to $time_hash for clarity.
		-Improve csv header titles.
		-Get rid of directory timestamp.
	
	prof_process --
		-Improve package name detecting regex.
		-Use iteration number for log and csv file basename instead of initial file creation timestamp.
		-Improve csv header titles.
		-Use just stderr for reaping data instead of stdout + stderr.

Affected files ...

.. //depot/projects/soc2007/revised_fbsd_pkgtools/usr/src/usr.sbin/pkg_install/prof_scripts/prof_postprocess.pl#3 edit
.. //depot/projects/soc2007/revised_fbsd_pkgtools/usr/src/usr.sbin/pkg_install/prof_scripts/prof_process.pl#4 edit
.. //depot/projects/soc2007/revised_fbsd_pkgtools/usr/src/usr.sbin/pkg_install/prof_scripts/run_prof.sh#6 edit

Differences ...

==== //depot/projects/soc2007/revised_fbsd_pkgtools/usr/src/usr.sbin/pkg_install/prof_scripts/prof_postprocess.pl#3 (text+ko) ====

@@ -1,7 +1,6 @@
 #!/usr/local/bin/perl -w
 
 use FileHandle;
-use Data::Dumper;
 
 if($#ARGV) {
         print "$0 [root_dir]\n";
@@ -10,16 +9,18 @@
 
 $root_dir = $ARGV[0];
 
+# Prefetch all directory names that need to be parsed over
 @dirs = `ls $root_dir`;
 
-$date_stamp = `/bin/date +'%F_%H.%M.%S'`;
+$summaries_dir = "$root_dir/summaries";
 
-chomp $date_stamp;
+if(-e $summaries_dir) {
+	print "Summary directory, $summaries_dir , already exists. Exiting..\n";
+	exit -1;
+} else {
+	system("mkdir -p $summaries_dir") || die "Couldn't create $summaries_dir\n";
+}
 
-$summaries_dir = "$root_dir/summaries.$date_stamp";
-
-system("mkdir -p $summaries_dir");
-
 for(my $i=0; $i <= $#dirs; $i++) {
 
 	next if($dirs[$i] =~ /summaries\..+/);
@@ -93,10 +94,6 @@
 
 	my @keys = sort(keys(%{shift()}));
 
-#	print Dumper(@_), "\n";
-
-#	print ( map "$_\n", @keys );
-
 	foreach my $key (@keys) {
 
 		for(my $i=0; $i <= $#{$header_fields_ref}; $i++) {
@@ -151,7 +148,7 @@
 	my $summary_fh = new FileHandle(">$summary_filename") || die "Filehandle couldn't be created for $summary_filename\n";
 
 	# Print out header..
-	print $summary_fh join(  ",", "Name", map ( "Median for $_,Standard deviation for $_,Minimum for $_,Maximum for $_" , @{$header_fields_ref} )  ) . "\n";
+	print $summary_fh join(  ",", "Name", map ( "$_ -- Mean,$_ -- Stdev,$_ -- Min,$_-- Max", @{$header_fields_ref} )  ) . "\n";
 
 	# Have to loop through once again to normalize the standard deviation to the number of elements, and print out the CSV formatted fields.
 	foreach my $key (@keys) {
@@ -171,6 +168,4 @@
 
 	print "Statistics summary printed to $summary_filename\n";
 
-#	print Dumper(%stdev_hash), "\n", Dumper(%mean_hash), "\n";
-
 }

==== //depot/projects/soc2007/revised_fbsd_pkgtools/usr/src/usr.sbin/pkg_install/prof_scripts/prof_process.pl#4 (text+ko) ====

@@ -5,19 +5,26 @@
 	exit 1;
 }
 
-$package = $ARGV[1];
-$package = $1 if $package =~ /(.+)-.+/;
+if($ARGV[1] =~ /(.+)-.+/) {
+	$package = $1;
+} else {
+	die "Package name format not understood. Is this actually derived from a package file?\n";
+}
+
 $logdir = "$ARGV[2]/$package";
 
-system("mkdir $logdir");
+system "mkdir $logdir" || die "Can't create $logdir\n";
+
+# Automatically pre-prime test machine by throwing out iteration 0 (most likely an outlying value -- package info not cached in vmem yet most likely..)
+system "pkg_add -f ${package}*.tbz";
 
 foreach my $iteration (1..$ARGV[0]) {
 
 	my ( %hash, %count_hash );
 
-	chomp ( $logfile_base = "$logdir/".`date +'%F-%H.%M.%S'` );
+	$logfile_base = "$logdir/$iteration";
 
-	system "/usr/bin/time -alh -o $logfile_base.log pkg_add -f ${package}*.tbz >> $logfile_base.log 2>> $logfile_base.log\n";
+	system "/usr/bin/time -alh -o $logfile_base.log pkg_add -f ${package}*.tbz 2> $logfile_base.log";
 
 	open IN_FH, "<$logfile_base.log" || die "Can't open log filehandle\n";
 
@@ -31,22 +38,22 @@
 
 		if($_ =~ /^\((\w+)\).+: ([\d\.]+)/) {
 
-			unless(exists($hash{$1})) {
-				$count_hash{$1} = 1;
-				$hash{$1} = 0 + $2;
-			} else {
-				$count_hash{$1}++;
-				$hash{$1} += $2;
+			unless(exists($time_hash{$1})) {
+				$count_hash{$1} = 0;
+				$time_hash{$1} = 0;
 			}
 
+			$count_hash{$1}++;
+			$time_hash{$1} += $2;
+
 		}
 
 	}
 
-	print OUT_FH "Function,number of iterations,time spent (secs)\n";
+	print OUT_FH "Function,iterations,time spent (secs)\n";
 
 	foreach(sort(keys(%hash))) {
-		printf OUT_FH "%s,%d,%4.20lf\n", $_, $count_hash{$_}, $hash{$_}; 
+		printf OUT_FH "%s,%d,%4.20lf\n", $_, $count_hash{$_}, $time_hash{$_}; 
 	}
 
 	close OUT_FH;

==== //depot/projects/soc2007/revised_fbsd_pkgtools/usr/src/usr.sbin/pkg_install/prof_scripts/run_prof.sh#6 (text+ko) ====

@@ -7,12 +7,13 @@
 fi
 
 ITERATIONS=500
-LOG_DIR="/store/motonoad0/new_profile_logs"
+LOG_DIR="/store/motonoad0/profile_logs/"
+LOG_DIR_STAMPED="$LOG_DIR/"`date +'%F_%H.%M.%S'`
 PACKAGE_DIR="/usr/ports/packages/All"
 
 SCRIPT_DIR=`realpath $0 | perl -e '$pwd=$ENV{PWD}; print "$1/\n" if <STDIN> =~ /(.+)\/[^\/]+$/;'`
 
-[ ! -d "$LOG_DIR" ] && mkdir -p $LOG_DIR
+[ ! -d "$LOG_DIR_STAMPED" ] && mkdir -p $LOG_DIR_STAMPED
 
 if [ ! -d "$PACKAGE_DIR" ] ; then
 	echo "ERROR: $PACKAGE_DIR doesn't exist; exiting.."
@@ -22,7 +23,7 @@
 cd $PACKAGE_DIR
 
 for i in $pkgs; do
-	$SCRIPT_DIR/prof_process.pl $ITERATIONS ${i}* $LOG_DIR
+	$SCRIPT_DIR/prof_process.pl $ITERATIONS ${i}* $LOG_DIR_STAMPED
 done
 
-$SCRIPT_DIR/prof_postprocess.pl $LOG_DIR
+$SCRIPT_DIR/prof_postprocess.pl $LOG_DIR_STAMPED


More information about the p4-projects mailing list