svn commit: r196101 - svnadmin/tools

Peter Wemm peter at FreeBSD.org
Tue Aug 11 21:23:32 UTC 2009


Author: peter
Date: Tue Aug 11 21:23:31 2009
New Revision: 196101
URL: http://svn.freebsd.org/changeset/base/196101

Log:
  WIP: split exporter into two halves, persistent driver that tracks the
  repository, and a backend that exports a single rev.

Modified:
  svnadmin/tools/export1.py
  svnadmin/tools/export2.py

Modified: svnadmin/tools/export1.py
==============================================================================
--- svnadmin/tools/export1.py	Tue Aug 11 21:21:20 2009	(r196100)
+++ svnadmin/tools/export1.py	Tue Aug 11 21:23:31 2009	(r196101)
@@ -10,103 +10,10 @@ import popen2
 import tempfile
 from svn import core, fs, delta, repos
 
-do_keywords = False
-
-# SVN's API structure is to do callbacks to a class to get notifications
-class ChangeReceiver(delta.Editor):
-  def __init__(self, fs_root, base_root, rev, fs_ptr, pool, do_kw):
-    self.fs_root = fs_root
-    self.base_root = base_root
-    self.fs_ptr = fs_ptr
-    self.rev = int(rev)
-    self.pool = pool
-    self.do_kw = do_kw
-    self.changes = []
-
-  def delete_entry(self, path, revision, parent_baton, pool):
-    self.changes.append(['D', path])
-
-  def add_file(self, path, parent_baton,
-               copyfrom_path, copyfrom_revision, file_pool):
-    self.changes.append(['A', path])
-    return [ '_', ' ', None ]
-
-  def open_file(self, path, parent_baton, base_revision, file_pool):
-    return [ '_', ' ', path ]
-
-  def apply_textdelta(self, file_baton, base_checksum):
-    text_mod, prop_mod, path = file_baton
-    file_baton[0] = 'U'
-    # no handler
-    return None
-
-  def change_file_prop(self, file_baton, name, value, pool):
-    text_mod, prop_mod, path = file_baton
-    if self.do_kw and name == core.SVN_PROP_KEYWORDS:
-      file_baton[1] = 'U'
-
-  def close_file(self, file_baton, text_checksum):
-    text_mod, prop_mod, path = file_baton
-    # test the path. it will be None if we added this file.
-    if path:
-      status = text_mod + prop_mod
-      # was there some kind of change?
-      if status != '_ ':
-        self.changes.append(['U', path])
-
-# Last path component
-def _basename(path):
-  idx = path.rfind('/')
-  if idx == -1:
-    return path   
-  return path[idx+1:]
-
-# Directory component
-def _dirname(path):
-  idx = path.rfind('/')
-  if idx == -1:
-    return ''
-  return path[:idx]
-
-# Keep track of best common prefix per directory group to keep commit time down
-class pathcollector():
-  def __init__(self):
-    self.paths = {}
-
-  # Not really a set..  calculates common prefix instead.
-  def __setitem__(self, prefix, dir):
-    # If we haven't seen a dir, start here
-    if not self.paths.has_key(prefix):
-      self.paths[prefix] = dir
-      return
-    # If it is the same dir, we're finished
-    if self.paths[prefix] == dir:
-      return
-    # See if we've found a common parent
-    parent = dir
-    while _dirname(parent) != parent:
-      parent = _dirname(parent)
-      # See if common prefix works
-      if self.paths[prefix] == parent:
-	return
-      # Raise common prefix
-      if self.paths[prefix].startswith(parent):
-	self.paths[prefix] = parent
-	return
-    print "WTF?"
-    assert false
-
-  def __iter__(self):
-    return self.paths.iteritems()
-
-# Issue a cvs command, aborting if a problem happens
-def do_cvs(cvspath, dir, cmd):
+# Issue an export command, aborting if a problem happens
+def do_export(cmd):
   ioerror = False
-  print("cvs path %s, dir %s, cmd %s" % (cvspath, dir, cmd))
-  cwd = os.getcwd()
-  os.chdir(os.path.join(cvspath, dir))
   pipe = popen2.Popen3(cmd)
-  os.chdir(cwd)
   output = ''
   try:
     output = pipe.fromchild.readlines()
@@ -116,256 +23,24 @@ def do_cvs(cvspath, dir, cmd):
     ioerror = True
   rv = pipe.wait()
   failed = (rv != 0) or ioerror
-  print 'Cvs output: ', output
+  print 'export2 output: ', output
   return failed
 
 # Dump a file from svn into cvs.  This has to apply the delta to the previous rev.
-def dump_file(fs_ptr, fs_root, rev, svnpath, cvspath, author, date, pool, workpath):
-  if do_keywords:
-    kw = fs.node_prop(fs_root, svnpath, core.SVN_PROP_KEYWORDS)
-    if not kw:
-      kw = ''
-    str = '$' + 'FreeBSD: %s %s %s %s $' % (cvspath, rev, date, author)
-    #str = '$' + 'FreeBSDId: %s %s %s %s $' % (cvspath, rev, date, author)
-    
-  subpool = core.svn_pool_create(pool)
-  stream = core.Stream(fs.file_contents(fs_root, svnpath, subpool))
-  str_list = []
-  while 1:
-    data = stream.read(core.SVN_STREAM_CHUNK_SIZE)
-    str_list.append(data)
-    if len(data) < core.SVN_STREAM_CHUNK_SIZE:
-      break
-  string = ''.join(str_list)
-  # Expand keywords
-  if do_keywords:
-    if kw == r'FreeBSD=%H':
-      old = '$' + 'FreeBSD$'
-      string = string.replace(old, str)
-  cvsfile = os.path.join(workpath, cvspath)
-#  sys.stdout.write('File contents:\n=========\n')
-#  sys.stdout.write(string)
-#  sys.stdout.write('=========\n')
-  executable = fs.node_prop(fs_root, svnpath, core.SVN_PROP_EXECUTABLE)
-  if executable:
-    mode = 0777
-  else:
-    mode = 0666
-  outfile = os.open(cvsfile, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode)
-  if not outfile:
-    sys.exit('cannot open %s for write' % cvsfile)
-  n = os.write(outfile, string)
-  if n != len(string):
-    sys.exit('short write. %d instead of %d' % (n, len(string)))
-  os.close(outfile)
-  core.svn_pool_destroy(subpool)
-
-# List of paths we export to cvs, and what branch tag (or None for HEAD)
-# Hard coded for now.  This will be automatically implied based on path transforms.
-maptable = [
-  # -current head
-  ( 'head/',        None ),
-  # -stable branches
-  ( 'stable/2.0.5/', 'RELENG_2_0_5' ),
-  ( 'stable/2.1/',   'RELENG_2_1_0' ),
-  ( 'stable/2.2/',   'RELENG_2_2' ),
-  ( 'stable/3/',     'RELENG_3' ),
-  ( 'stable/4/',     'RELENG_4' ),
-  ( 'stable/5/',     'RELENG_5' ),
-  ( 'stable/6/',     'RELENG_6' ),
-  ( 'stable/7/',     'RELENG_7' ),
-  # errata / security / releng branches
-  ( 'releng/ALPHA_2_0/','ALPHA_2_0' ),
-  ( 'releng/BETA_2_0/', 'BETA_2_0' ),
-  ( 'releng/4.3/',   'RELENG_4_3' ),
-  ( 'releng/4.4/',   'RELENG_4_4' ),
-  ( 'releng/4.5/',   'RELENG_4_5' ),
-  ( 'releng/4.6/',   'RELENG_4_6' ),
-  ( 'releng/4.7/',   'RELENG_4_7' ),
-  ( 'releng/4.8/',   'RELENG_4_8' ),
-  ( 'releng/4.9/',   'RELENG_4_9' ),
-  ( 'releng/4.10/',  'RELENG_4_10' ),
-  ( 'releng/4.11/',  'RELENG_4_11' ),
-  ( 'releng/4.12/',  'RELENG_4_12' ),
-  ( 'releng/5.0/',   'RELENG_5_0' ),
-  ( 'releng/5.1/',   'RELENG_5_1' ),
-  ( 'releng/5.2/',   'RELENG_5_2' ),
-  ( 'releng/5.3/',   'RELENG_5_3' ),
-  ( 'releng/5.4/',   'RELENG_5_4' ),
-  ( 'releng/5.5/',   'RELENG_5_5' ),
-  ( 'releng/5.6/',   'RELENG_5_6' ),
-  ( 'releng/6.0/',   'RELENG_6_0' ),
-  ( 'releng/6.1/',   'RELENG_6_1' ),
-  ( 'releng/6.2/',   'RELENG_6_2' ),
-  ( 'releng/6.3/',   'RELENG_6_3' ),
-  ( 'releng/6.4/',   'RELENG_6_4' ),
-  ( 'releng/7.0/',   'RELENG_7_0' ),
-  ( 'releng/7.1/',   'RELENG_7_1' ),
-  ( 'releng/7.2/',   'RELENG_7_2' ),
-]
-
-def map2cvs(svnpath):
-  for prefix, branch in maptable:
-    plen = len(prefix)
-    if svnpath.startswith(prefix):
-      return 'src/' + svnpath[plen:], branch
-  return None, None
-
-# List of special remap cases.
-roottable = [
-  ( 'svnadmin/conf/access',  'CVSROOT/access',  'CVSROOT' ),
-  ( 'svnadmin/conf/mentors', 'CVSROOT/mentors', 'CVSROOT' ),
-]
-
-def maproot(p):
-  for svnpath, path, dir in roottable:
-    if p == svnpath:
-      return path, dir
-  return None, None
-
-# Add intermediate directories to the cvs checkout area as needed.
-# XXX should use 'cvs update -d -l' if the dir exists in cvsroot
-def makedirs(cvspath, path, base):
-  #print 'Makedirs:', cvspath, path
-  if not path.startswith(base):
-    sys.exit('Illegal path %s' % path)
-  if path == base:
-    return
-  makedirs(cvspath, _dirname(path), base)
-  fullpath = os.path.join(cvspath, path)
-  if os.path.isfile(fullpath):
-     sys.exit('Dest dir is a file' % path)
-  if not os.path.isdir(fullpath):
-    try:
-      #print "Making directory " + fullpath
-      os.makedirs(fullpath)
-      failed = do_cvs(cvspath, _dirname(path), "cvs -q add %s" % _basename(path))
-      assert not failed
-    except OSError:
-      sys.exit('Cannot mkdir %s' % path)
-  #print 'Dirpath complete: ' + path
-
-# Export a single change to cvs.
-def exportrev(pool, fs_ptr, rev, cvspath):
-  def authz_cb(root, path, pool):
-    return True
-
-  subpool = core.svn_pool_create(pool)
-  # Connect up to the revision
-  fs_root = fs.revision_root(fs_ptr, rev, subpool)
-  base_root = fs.revision_root(fs_ptr, rev - 1, subpool)
-  editor = ChangeReceiver(fs_root, base_root, rev, fs_ptr, subpool, do_keywords)
-  e_ptr, e_baton = delta.make_editor(editor, subpool)
-  repos.dir_delta(base_root, '', '', fs_root, '', e_ptr, e_baton, authz_cb, 0, 1, 0, 0, subpool)
-
-  # Author
-  author = fs.revision_prop(fs_ptr, rev, core.SVN_PROP_REVISION_AUTHOR)
-  if not author:
-    author = 'NoAuthor'
-  if author == 'davidg':
-    author = 'dg'
-  os.environ['CVS_AUTHOR'] = author
-  print 'Author: ' + author
-
-  # Date
-  date = fs.revision_prop(fs_ptr, rev, core.SVN_PROP_REVISION_DATE)
-  if date:
-    aprtime = core.svn_time_from_cstring(date)
-    secs = aprtime / 1000000  # aprtime is microseconds; make seconds
-    tm = time.gmtime(secs)
-    date = time.strftime('%Y-%m-%d %H:%M:%SZ', tm)
-    os.environ['CVS_TIMESTAMP'] = "%d" % secs
-  else:
-    date = 'NoDate'
-    if os.environ.has_key('CVS_TIMESTAMP'):
-      del os.environ['CVS_TIMESTAMP']
-
-  # Build log message to export
-  cvslog = 'SVN rev %d on %s by %s\n' % (rev, date, author)
-  svnlog = fs.revision_prop(fs_ptr, rev, core.SVN_PROP_REVISION_LOG)
-  if svnlog:
-    cvslog += '\n' + svnlog
-
-  pc = pathcollector()
-
-  for k, p in editor.changes:
-    #print 'Path ', p
-    # Hack, hack
-    (path, dir) = maproot(p)
-    if path:
-      workpath = cvspath
-      dump_file(fs_ptr, fs_root, rev, p, path, author, date, subpool, workpath)
-      pc[workpath] = dir
-      continue
-    (path, tag) = map2cvs(p)
-    if not path:
-      continue
-    if tag:
-      workpath = os.path.join(cvspath, tag)
-      uptag = '-r ' + tag
-    else:
-      workpath = cvspath
-      uptag = '-A'
-    #print workpath
-    if not os.path.isdir(workpath):
-      os.makedirs(workpath)
-    if not os.path.isdir(os.path.join(workpath, 'src')):
-      failed = do_cvs(workpath, '', "cvs -Rq co %s src" % uptag)
-      assert not failed
-    # at this point, the top directory and /src should exist
-    print p, path, k
-    # hacks
-    #if p == 'head/contrib/file/FREEBSD-upgrade' and k == 'A':
-    #  continue
-    #if p == 'head/contrib/file/magic2mime' and k == 'A':
-    #  continue
-    #if p == 'head/lib/libc/stdio/asprintf.c' and k == 'D':
-    #  continue
-    if p == 'head/tools/build/options/WITH_BIND_LIBS' and k == 'A' and rev == 193280:
-      continue
-    makedirs(workpath, _dirname(path), 'src')
-    # Now the directory for the files must exist, and branch tag will be sticky
-    assert os.path.isdir(os.path.join(workpath, _dirname(path)))
-    assert k == 'A' or k == 'U' or k == 'D'
-
-    if k == 'A' or k == 'U':
-      print 'add/update file ' + path + '.'
-      destpath = os.path.join(workpath, path)
-      existed = os.path.isfile(destpath)
-      dump_file(fs_ptr, fs_root, rev, p, path, author, date, subpool, workpath)
-      if not existed:
-	print 'cvs add file ' + path + '.'
-	failed = do_cvs(workpath, _dirname(path), "cvs -q add %s" % _basename(path))
-	assert not failed
-    elif k == 'D':
-      print 'cvs rm -f file ' + path + '.'
-      failed = do_cvs(workpath, _dirname(path), "cvs -q rm -f %s" % _basename(path))
-      assert not failed
-    pc[workpath] = _dirname(path)
-
-  # aggregate the commit
-  for root, dir in pc:
-    fd, logfile = tempfile.mkstemp()
-    os.write(fd, cvslog)
-    os.close(fd)
-    failed = do_cvs(root, dir, "cvs -q commit -F %s" % logfile)
-    assert not failed
-    os.remove(logfile)
-  core.svn_pool_destroy(subpool)
-
 # Loop for the export range
-def export(pool, repos_path, cvspath):
+def export(pool, repos_path, cvspath, cvsroot):
   repos_path = core.svn_path_canonicalize(repos_path)
   fs_ptr = repos.fs(repos.open(repos_path, pool))
   while True:
     curr_rev = fs.youngest_rev(fs_ptr)
     last_rev = int(fs.revision_prop(fs_ptr, 0, 'fbsd:lastexp'))
     if last_rev < curr_rev:
-      time.sleep(5)
       print '%d %s' % (last_rev, curr_rev)
       rev = '%d' % (last_rev + 1)
       print '==========> export rev ' + rev
-      exportrev(pool, fs_ptr, last_rev + 1, cvspath)
+      cmd = './export2.py %s %s %d %s' % (repos_path, cvspath, last_rev + 1, cvsroot)
+      failed = do_export(cmd)
+      assert not failed
       fs.change_rev_prop(fs_ptr, 0, 'fbsd:lastexp', rev)
       continue
     print "."
@@ -374,8 +49,4 @@ def export(pool, repos_path, cvspath):
 
 if __name__ == '__main__':
   print "Version: $FreeBSD$"
-  os.environ['CVSROOT'] = '/r/ncvs'
-  core.run_app(export, '/r/svnmirror/base', '/r/svn2cvs/cvs')
-  # test rig
-  #os.environ['CVSROOT'] = '/home/peter/exp/cvs'
-  #core.run_app(export, '/home/peter/exp/svn', '/home/peter/exp/co')
+  core.run_app(export, '/r/svnmirror/base', '/r/svn2cvs/cvs', '/r/ncvs')

Modified: svnadmin/tools/export2.py
==============================================================================
--- svnadmin/tools/export2.py	Tue Aug 11 21:21:20 2009	(r196100)
+++ svnadmin/tools/export2.py	Tue Aug 11 21:23:31 2009	(r196101)
@@ -354,28 +354,14 @@ def exportrev(pool, fs_ptr, rev, cvspath
   core.svn_pool_destroy(subpool)
 
 # Loop for the export range
-def export(pool, repos_path, cvspath):
+def export(pool, repos_path, cvspath, rev):
   repos_path = core.svn_path_canonicalize(repos_path)
   fs_ptr = repos.fs(repos.open(repos_path, pool))
-  while True:
-    curr_rev = fs.youngest_rev(fs_ptr)
-    last_rev = int(fs.revision_prop(fs_ptr, 0, 'fbsd:lastexp'))
-    if last_rev < curr_rev:
-      time.sleep(5)
-      print '%d %s' % (last_rev, curr_rev)
-      rev = '%d' % (last_rev + 1)
-      print '==========> export rev ' + rev
-      exportrev(pool, fs_ptr, last_rev + 1, cvspath)
-      fs.change_rev_prop(fs_ptr, 0, 'fbsd:lastexp', rev)
-      continue
-    print "."
-    time.sleep(15)
-
+  print '==========> export rev ' + rev
+  exportrev(pool, fs_ptr, rev, cvspath)
 
 if __name__ == '__main__':
   print "Version: $FreeBSD$"
-  os.environ['CVSROOT'] = '/r/ncvs'
-  core.run_app(export, '/r/svnmirror/base', '/r/svn2cvs/cvs')
-  # test rig
-  #os.environ['CVSROOT'] = '/home/peter/exp/cvs'
-  #core.run_app(export, '/home/peter/exp/svn', '/home/peter/exp/co')
+  assert len(sys.argv) == 5
+  os.environ['CVSROOT'] = sys.argv[4]
+  core.run_app(export, sys.argv[1], sys.argv[2], int(sys.argv[3]))


More information about the svn-src-svnadmin mailing list