import os
import re
import sys
import glob
import shutil
import urllib2
import hashlib
import tarfile
import logging
import datetime
import operator
import itertools
import subprocess
import argparse
try:
import ezt
except ImportError:
ezt_path = os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0])))
ezt_path = os.path.join(ezt_path, 'build', 'generator')
sys.path.append(ezt_path)
import ezt
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'
repos = 'http://svn.apache.org/repos/asf/subversion'
people_host = 'minotaur.apache.org'
people_dist_dir = '/www/www.apache.org/dist/subversion'
class Version(object):
regex = re.compile('(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')
def __init__(self, ver_str):
match = self.regex.search(ver_str)
if not match:
raise RuntimeError("Bad version string '%s'" % ver_str)
self.major = int(match.group(1))
self.minor = int(match.group(2))
self.patch = int(match.group(3))
if match.group(4):
self.pre = match.group(4)
self.pre_num = int(match.group(5))
else:
self.pre = None
self.pre_num = None
self.base = '%d.%d.%d' % (self.major, self.minor, self.patch)
def is_prerelease(self):
return self.pre != None
def __lt__(self, that):
if self.major < that.major: return True
if self.major > that.major: return False
if self.minor < that.minor: return True
if self.minor > that.minor: return False
if self.patch < that.patch: return True
if self.patch > that.patch: return False
if not self.pre and not that.pre: return False
if not self.pre and that.pre: return False
if self.pre and not that.pre: return True
if self.pre != that.pre:
return self.pre < that.pre
else:
return self.pre_num < that.pre_num
def __str(self):
if self.pre:
extra = '-%s%d' % (self.pre, self.pre_num)
else:
extra = ''
return self.base + extra
def __repr__(self):
return "Version('%s')" % self.__str()
def __str__(self):
return self.__str()
def get_prefix(base_dir):
return os.path.join(base_dir, 'prefix')
def get_tempdir(base_dir):
return os.path.join(base_dir, 'tempdir')
def get_deploydir(base_dir):
return os.path.join(base_dir, 'deploy')
def get_tmpldir():
return os.path.join(os.path.abspath(sys.path[0]), 'templates')
def get_tmplfile(filename):
try:
return open(os.path.join(get_tmpldir(), filename))
except IOError:
return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename)
def get_nullfile():
return open('/dev/null', 'w')
def run_script(verbose, script):
if verbose:
stdout = None
stderr = None
else:
stdout = get_nullfile()
stderr = subprocess.STDOUT
for l in script.split('\n'):
subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)
def download_file(url, target):
response = urllib2.urlopen(url)
target_file = open(target, 'w')
target_file.write(response.read())
def assert_people():
if os.uname()[1] != people_host:
raise RuntimeError('Not running on expected host "%s"' % people_host)
def cleanup(args):
'Remove generated files and folders.'
logging.info('Cleaning')
shutil.rmtree(get_prefix(args.base_dir), True)
shutil.rmtree(get_tempdir(args.base_dir), True)
shutil.rmtree(get_deploydir(args.base_dir), True)
class RollDep(object):
'The super class for each of the build dependencies.'
def __init__(self, base_dir, use_existing, verbose):
self._base_dir = base_dir
self._use_existing = use_existing
self._verbose = verbose
def _test_version(self, cmd):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
(stdout, stderr) = proc.communicate()
rc = proc.wait()
if rc: return ''
return stdout.split('\n')
def build(self):
if not hasattr(self, '_extra_configure_flags'):
self._extra_configure_flags = ''
cwd = os.getcwd()
tempdir = get_tempdir(self._base_dir)
tarball = os.path.join(tempdir, self._filebase + '.tar.gz')
if os.path.exists(tarball):
if not self._use_existing:
raise RuntimeError('autoconf tarball "%s" already exists'
% tarball)
logging.info('Using existing %s.tar.gz' % self._filebase)
else:
logging.info('Fetching %s' % self._filebase)
download_file(self._url, tarball)
tarfile.open(tarball).extractall(tempdir)
logging.info('Building ' + self.label)
os.chdir(os.path.join(tempdir, self._filebase))
run_script(self._verbose,
'''./configure --prefix=%s %s
make
make install''' % (get_prefix(self._base_dir),
self._extra_configure_flags))
os.chdir(cwd)
class AutoconfDep(RollDep):
def __init__(self, base_dir, use_existing, verbose):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'autoconf'
self._filebase = 'autoconf-' + autoconf_ver
self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase
def have_usable(self):
output = self._test_version(['autoconf', '-V'])
if not output: return False
version = output[0].split()[-1:][0]
return version == autoconf_ver
def use_system(self):
if not self._use_existing: return False
return self.have_usable()
class LibtoolDep(RollDep):
def __init__(self, base_dir, use_existing, verbose):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'libtool'
self._filebase = 'libtool-' + libtool_ver
self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase
def have_usable(self):
output = self._test_version(['libtool', '--version'])
if not output: return False
version = output[0].split()[-1:][0]
return version == libtool_ver
def use_system(self):
return False
class SwigDep(RollDep):
def __init__(self, base_dir, use_existing, verbose, sf_mirror):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'swig'
self._filebase = 'swig-' + swig_ver
self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
{ 'swig' : self._filebase,
'sf_mirror' : sf_mirror }
self._extra_configure_flags = '--without-pcre'
def have_usable(self):
output = self._test_version(['swig', '-version'])
if not output: return False
version = output[1].split()[-1:][0]
return version == swig_ver
def use_system(self):
if not self._use_existing: return False
return self.have_usable()
def build_env(args):
'Download prerequisites for a release and prepare the environment.'
logging.info('Creating release environment')
try:
os.mkdir(get_prefix(args.base_dir))
os.mkdir(get_tempdir(args.base_dir))
except OSError:
if not args.use_existing:
raise
autoconf = AutoconfDep(args.base_dir, args.use_existing, args.verbose)
libtool = LibtoolDep(args.base_dir, args.use_existing, args.verbose)
swig = SwigDep(args.base_dir, args.use_existing, args.verbose,
args.sf_mirror)
for dep in [autoconf, libtool, swig]:
if dep.use_system():
logging.info('Using system %s' % dep.label)
else:
dep.build()
def roll_tarballs(args):
'Create the release artifacts.'
extns = ['zip', 'tar.gz', 'tar.bz2']
if args.branch:
branch = args.branch
else:
branch = args.version.base[:-1] + 'x'
logging.info('Rolling release %s from branch %s@%d' % (args.version,
branch, args.revnum))
autoconf = AutoconfDep(args.base_dir, False, args.verbose)
libtool = LibtoolDep(args.base_dir, False, args.verbose)
swig = SwigDep(args.base_dir, False, args.verbose, None)
for dep in [autoconf, libtool, swig]:
if not dep.have_usable():
raise RuntimeError('Cannot find usable %s' % dep.label)
if branch != 'trunk':
trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
args.revnum)
proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
trunk_CHANGES],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
(stdout, stderr) = proc.communicate()
proc.wait()
if stdout:
raise RuntimeError('CHANGES not synced between trunk and branch')
if not os.path.exists(get_deploydir(args.base_dir)):
os.mkdir(get_deploydir(args.base_dir))
extra_args = ''
if args.version.is_prerelease():
extra_args = '-%s %d' % (args.version.pre, args.version.pre_num)
logging.info('Building UNIX tarballs')
run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
% (sys.path[0], args.version.base, branch, args.revnum,
extra_args) )
logging.info('Buildling Windows tarballs')
run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
% (sys.path[0], args.version.base, branch, args.revnum,
extra_args) )
logging.info('Moving artifacts and calculating checksums')
for e in extns:
if args.version.pre == 'nightly':
filename = 'subversion-trunk.%s' % e
else:
filename = 'subversion-%s.%s' % (args.version, e)
shutil.move(filename, get_deploydir(args.base_dir))
filename = os.path.join(get_deploydir(args.base_dir), filename)
m = hashlib.sha1()
m.update(open(filename, 'r').read())
open(filename + '.sha1', 'w').write(m.hexdigest())
shutil.move('svn_version.h.dist', get_deploydir(args.base_dir))
def post_candidates(args):
'Post the generated tarballs to web-accessible directory.'
if args.target:
target = args.target
else:
target = os.path.join(os.getenv('HOME'), 'public_html', 'svn',
args.version)
if args.code_name:
dirname = args.code_name
else:
dirname = 'deploy'
if not os.path.exists(target):
os.makedirs(target)
data = { 'version' : args.version,
'revnum' : args.revnum,
'dirname' : dirname,
}
if args.version.is_prerelease():
if args.version.pre == 'nightly':
template_filename = 'nightly-candidates.ezt'
else:
template_filename = 'rc-candidates.ezt'
else:
template_filename = 'stable-candidates.ezt'
template = ezt.Template()
template.parse(get_tmplfile(template_filename).read())
template.generate(open(os.path.join(target, 'index.html'), 'w'), data)
logging.info('Moving tarballs to %s' % os.path.join(target, dirname))
if os.path.exists(os.path.join(target, dirname)):
shutil.rmtree(os.path.join(target, dirname))
shutil.copytree(get_deploydir(args.base_dir), os.path.join(target, dirname))
def clean_dist(args):
'Clean the distribution directory of all but the most recent artifacts.'
regex = re.compile('subversion-(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')
if not args.dist_dir:
assert_people()
args.dist_dir = people_dist_dir
logging.info('Cleaning dist dir \'%s\'' % args.dist_dir)
filenames = glob.glob(os.path.join(args.dist_dir, 'subversion-*.tar.gz'))
versions = []
for filename in filenames:
versions.append(Version(filename))
for k, g in itertools.groupby(sorted(versions),
lambda x: (x.major, x.minor)):
releases = list(g)
logging.info("Saving release '%s'", releases[-1])
for r in releases[:-1]:
for filename in glob.glob(os.path.join(args.dist_dir,
'subversion-%s.*' % r)):
logging.info("Removing '%s'" % filename)
os.remove(filename)
def write_news(args):
'Write text for the Subversion website.'
data = { 'date' : datetime.date.today().strftime('%Y%m%d'),
'date_pres' : datetime.date.today().strftime('%Y-%m-%d'),
'version' : str(args.version),
'version_base' : args.version.base,
}
if args.version.is_prerelease():
template_filename = 'rc-news.ezt'
else:
template_filename = 'stable-news.ezt'
template = ezt.Template()
template.parse(get_tmplfile(template_filename).read())
template.generate(sys.stdout, data)
def get_sha1info(args):
'Return a list of sha1 info for the release'
sha1s = glob.glob(os.path.join(get_deploydir(args.base_dir), '*.sha1'))
class info(object):
pass
sha1info = []
for s in sha1s:
i = info()
i.filename = os.path.basename(s)[:-5]
i.sha1 = open(s, 'r').read()
sha1info.append(i)
return sha1info
def write_announcement(args):
'Write the release announcement.'
sha1info = get_sha1info(args)
data = { 'version' : args.version,
'sha1info' : sha1info,
'siginfo' : open('getsigs-output', 'r').read(),
'major-minor' : args.version.base[:3],
'major-minor-patch' : args.version.base,
}
if args.version.is_prerelease():
template_filename = 'rc-release-ann.ezt'
else:
template_filename = 'stable-release-ann.ezt'
template = ezt.Template(compress_whitespace = False)
template.parse(get_tmplfile(template_filename).read())
template.generate(sys.stdout, data)
def main():
'Parse arguments, and drive the appropriate subcommand.'
parser = argparse.ArgumentParser(
description='Create an Apache Subversion release.')
parser.add_argument('--clean', action='store_true', default=False,
help='Remove any directories previously created by %(prog)s')
parser.add_argument('--verbose', action='store_true', default=False,
help='Increase output verbosity')
parser.add_argument('--base-dir', default=os.getcwd(),
help='''The directory in which to create needed files and
folders. The default is the current working
directory.''')
subparsers = parser.add_subparsers(title='subcommands')
subparser = subparsers.add_parser('build-env',
help='''Download release prerequisistes, including autoconf,
libtool, and swig.''')
subparser.set_defaults(func=build_env)
subparser.add_argument('--sf-mirror', default='softlayer',
help='''The mirror to use for downloading files from
SourceForge. If in the EU, you may want to use
'kent' for this value.''')
subparser.add_argument('--use-existing', action='store_true', default=False,
help='''Attempt to use existing build dependencies before
downloading and building a private set.''')
subparser = subparsers.add_parser('roll',
help='''Create the release artifacts.''')
subparser.set_defaults(func=roll_tarballs)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('revnum', type=int,
help='''The revision number to base the release on.''')
subparser.add_argument('--branch',
help='''The branch to base the release on.''')
subparser = subparsers.add_parser('post-candidates',
help='''Build the website to host the candidate tarballs.
The default location is somewhere in ~/public_html.
''')
subparser.set_defaults(func=post_candidates)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('revnum', type=int,
help='''The revision number to base the release on.''')
subparser.add_argument('--target',
help='''The full path to the destination.''')
subparser.add_argument('--code-name',
help='''A whimsical name for the release, used only for
naming the download directory.''')
subparser = subparsers.add_parser('clean-dist',
help='''Clean the distribution directory (and mirrors) of
all but the most recent MAJOR.MINOR release. If no
dist-dir is given, this command will assume it is
running on people.apache.org.''')
subparser.set_defaults(func=clean_dist)
subparser.add_argument('--dist-dir',
help='''The directory to clean.''')
subparser = subparsers.add_parser('write-news',
help='''Output to stdout template text for use in the news
section of the Subversion website.''')
subparser.set_defaults(func=write_news)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser = subparsers.add_parser('write-announcement',
help='''Output to stdout template text for the emailed
release announcement.''')
subparser.set_defaults(func=write_announcement)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser = subparsers.add_parser('clean',
help='''The same as the '--clean' switch, but as a
separate subcommand.''')
subparser.set_defaults(func=cleanup)
args = parser.parse_args()
if args.clean:
cleanup(args)
logger = logging.getLogger()
if args.verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
+ os.environ['PATH']
args.func(args)
if __name__ == '__main__':
main()