#!/usr/bin/env python # # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # About this script: # This script is intended to simplify creating Subversion releases, by # automating as much as is possible. It works well with our Apache # infrastructure, and should make rolling, posting, and announcing # releases dirt simple. # # This script may be run on a number of platforms, but it is intended to # be run on people.apache.org. As such, it may have dependencies (such # as Python version) which may not be common, but are guaranteed to be # available on people.apache.org. # It'd be kind of nice to use the Subversion python bindings in this script, # but people.apache.org doesn't currently have them installed # Stuff we need import os import re import sys import glob import shutil import urllib2 import hashlib import tarfile import logging import datetime import operator import itertools import subprocess import argparse # standard in Python 2.7 # Find ezt, using Subversion's copy, if there isn't one on the system. try: import ezt except ImportError: ezt_path = os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0]))) ezt_path = os.path.join(ezt_path, 'build', 'generator') sys.path.append(ezt_path) import ezt # Our required / recommended versions autoconf_ver = '2.68' libtool_ver = '2.4' swig_ver = '2.0.4' # Some constants repos = 'http://svn.apache.org/repos/asf/subversion' people_host = 'minotaur.apache.org' people_dist_dir = '/www/www.apache.org/dist/subversion' #---------------------------------------------------------------------- # Utility functions class Version(object): regex = re.compile('(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?') def __init__(self, ver_str): match = self.regex.search(ver_str) if not match: raise RuntimeError("Bad version string '%s'" % ver_str) self.major = int(match.group(1)) self.minor = int(match.group(2)) self.patch = int(match.group(3)) if match.group(4): self.pre = match.group(4) self.pre_num = int(match.group(5)) else: self.pre = None self.pre_num = None self.base = '%d.%d.%d' % (self.major, self.minor, self.patch) def is_prerelease(self): return self.pre != None def __lt__(self, that): if self.major < that.major: return True if self.major > that.major: return False if self.minor < that.minor: return True if self.minor > that.minor: return False if self.patch < that.patch: return True if self.patch > that.patch: return False if not self.pre and not that.pre: return False if not self.pre and that.pre: return False if self.pre and not that.pre: return True # We are both pre-releases if self.pre != that.pre: return self.pre < that.pre else: return self.pre_num < that.pre_num def __str(self): if self.pre: extra = '-%s%d' % (self.pre, self.pre_num) else: extra = '' return self.base + extra def __repr__(self): return "Version('%s')" % self.__str() def __str__(self): return self.__str() def get_prefix(base_dir): return os.path.join(base_dir, 'prefix') def get_tempdir(base_dir): return os.path.join(base_dir, 'tempdir') def get_deploydir(base_dir): return os.path.join(base_dir, 'deploy') def get_tmpldir(): return os.path.join(os.path.abspath(sys.path[0]), 'templates') def get_tmplfile(filename): try: return open(os.path.join(get_tmpldir(), filename)) except IOError: # Hmm, we had a problem with the local version, let's try the repo return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename) def get_nullfile(): # This is certainly not cross platform return open('/dev/null', 'w') def run_script(verbose, script): if verbose: stdout = None stderr = None else: stdout = get_nullfile() stderr = subprocess.STDOUT for l in script.split('\n'): subprocess.check_call(l.split(), stdout=stdout, stderr=stderr) def download_file(url, target): response = urllib2.urlopen(url) target_file = open(target, 'w') target_file.write(response.read()) def assert_people(): if os.uname()[1] != people_host: raise RuntimeError('Not running on expected host "%s"' % people_host) #---------------------------------------------------------------------- # Cleaning up the environment def cleanup(args): 'Remove generated files and folders.' logging.info('Cleaning') shutil.rmtree(get_prefix(args.base_dir), True) shutil.rmtree(get_tempdir(args.base_dir), True) shutil.rmtree(get_deploydir(args.base_dir), True) #---------------------------------------------------------------------- # Creating an environment to roll the release class RollDep(object): 'The super class for each of the build dependencies.' def __init__(self, base_dir, use_existing, verbose): self._base_dir = base_dir self._use_existing = use_existing self._verbose = verbose def _test_version(self, cmd): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) (stdout, stderr) = proc.communicate() rc = proc.wait() if rc: return '' return stdout.split('\n') def build(self): if not hasattr(self, '_extra_configure_flags'): self._extra_configure_flags = '' cwd = os.getcwd() tempdir = get_tempdir(self._base_dir) tarball = os.path.join(tempdir, self._filebase + '.tar.gz') if os.path.exists(tarball): if not self._use_existing: raise RuntimeError('autoconf tarball "%s" already exists' % tarball) logging.info('Using existing %s.tar.gz' % self._filebase) else: logging.info('Fetching %s' % self._filebase) download_file(self._url, tarball) # Extract tarball tarfile.open(tarball).extractall(tempdir) logging.info('Building ' + self.label) os.chdir(os.path.join(tempdir, self._filebase)) run_script(self._verbose, '''./configure --prefix=%s %s make make install''' % (get_prefix(self._base_dir), self._extra_configure_flags)) os.chdir(cwd) class AutoconfDep(RollDep): def __init__(self, base_dir, use_existing, verbose): RollDep.__init__(self, base_dir, use_existing, verbose) self.label = 'autoconf' self._filebase = 'autoconf-' + autoconf_ver self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase def have_usable(self): output = self._test_version(['autoconf', '-V']) if not output: return False version = output[0].split()[-1:][0] return version == autoconf_ver def use_system(self): if not self._use_existing: return False return self.have_usable() class LibtoolDep(RollDep): def __init__(self, base_dir, use_existing, verbose): RollDep.__init__(self, base_dir, use_existing, verbose) self.label = 'libtool' self._filebase = 'libtool-' + libtool_ver self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase def have_usable(self): output = self._test_version(['libtool', '--version']) if not output: return False version = output[0].split()[-1:][0] return version == libtool_ver def use_system(self): # We unconditionally return False here, to avoid using a borked # system libtool (I'm looking at you, Debian). return False class SwigDep(RollDep): def __init__(self, base_dir, use_existing, verbose, sf_mirror): RollDep.__init__(self, base_dir, use_existing, verbose) self.label = 'swig' self._filebase = 'swig-' + swig_ver self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \ { 'swig' : self._filebase, 'sf_mirror' : sf_mirror } self._extra_configure_flags = '--without-pcre' def have_usable(self): output = self._test_version(['swig', '-version']) if not output: return False version = output[1].split()[-1:][0] return version == swig_ver def use_system(self): if not self._use_existing: return False return self.have_usable() def build_env(args): 'Download prerequisites for a release and prepare the environment.' logging.info('Creating release environment') try: os.mkdir(get_prefix(args.base_dir)) os.mkdir(get_tempdir(args.base_dir)) except OSError: if not args.use_existing: raise autoconf = AutoconfDep(args.base_dir, args.use_existing, args.verbose) libtool = LibtoolDep(args.base_dir, args.use_existing, args.verbose) swig = SwigDep(args.base_dir, args.use_existing, args.verbose, args.sf_mirror) # iterate over our rolling deps, and build them if needed for dep in [autoconf, libtool, swig]: if dep.use_system(): logging.info('Using system %s' % dep.label) else: dep.build() #---------------------------------------------------------------------- # Create release artifacts def roll_tarballs(args): 'Create the release artifacts.' extns = ['zip', 'tar.gz', 'tar.bz2'] if args.branch: branch = args.branch else: branch = args.version.base[:-1] + 'x' logging.info('Rolling release %s from branch %s@%d' % (args.version, branch, args.revnum)) # Ensure we've got the appropriate rolling dependencies available autoconf = AutoconfDep(args.base_dir, False, args.verbose) libtool = LibtoolDep(args.base_dir, False, args.verbose) swig = SwigDep(args.base_dir, False, args.verbose, None) for dep in [autoconf, libtool, swig]: if not dep.have_usable(): raise RuntimeError('Cannot find usable %s' % dep.label) # Make sure CHANGES is sync'd if branch != 'trunk': trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum) branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch, args.revnum) proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES, trunk_CHANGES], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) (stdout, stderr) = proc.communicate() proc.wait() if stdout: raise RuntimeError('CHANGES not synced between trunk and branch') # Create the output directory if not os.path.exists(get_deploydir(args.base_dir)): os.mkdir(get_deploydir(args.base_dir)) # For now, just delegate to dist.sh to create the actual artifacts extra_args = '' if args.version.is_prerelease(): extra_args = '-%s %d' % (args.version.pre, args.version.pre_num) logging.info('Building UNIX tarballs') run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s' % (sys.path[0], args.version.base, branch, args.revnum, extra_args) ) logging.info('Buildling Windows tarballs') run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s' % (sys.path[0], args.version.base, branch, args.revnum, extra_args) ) # Move the results to the deploy directory logging.info('Moving artifacts and calculating checksums') for e in extns: if args.version.pre == 'nightly': filename = 'subversion-trunk.%s' % e else: filename = 'subversion-%s.%s' % (args.version, e) shutil.move(filename, get_deploydir(args.base_dir)) filename = os.path.join(get_deploydir(args.base_dir), filename) m = hashlib.sha1() m.update(open(filename, 'r').read()) open(filename + '.sha1', 'w').write(m.hexdigest()) shutil.move('svn_version.h.dist', get_deploydir(args.base_dir)) # And we're done! #---------------------------------------------------------------------- # Post the candidate release artifacts def post_candidates(args): 'Post the generated tarballs to web-accessible directory.' if args.target: target = args.target else: target = os.path.join(os.getenv('HOME'), 'public_html', 'svn', args.version) if args.code_name: dirname = args.code_name else: dirname = 'deploy' if not os.path.exists(target): os.makedirs(target) data = { 'version' : args.version, 'revnum' : args.revnum, 'dirname' : dirname, } # Choose the right template text if args.version.is_prerelease(): if args.version.pre == 'nightly': template_filename = 'nightly-candidates.ezt' else: template_filename = 'rc-candidates.ezt' else: template_filename = 'stable-candidates.ezt' template = ezt.Template() template.parse(get_tmplfile(template_filename).read()) template.generate(open(os.path.join(target, 'index.html'), 'w'), data) logging.info('Moving tarballs to %s' % os.path.join(target, dirname)) if os.path.exists(os.path.join(target, dirname)): shutil.rmtree(os.path.join(target, dirname)) shutil.copytree(get_deploydir(args.base_dir), os.path.join(target, dirname)) #---------------------------------------------------------------------- # Clean dist def clean_dist(args): 'Clean the distribution directory of all but the most recent artifacts.' regex = re.compile('subversion-(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?') if not args.dist_dir: assert_people() args.dist_dir = people_dist_dir logging.info('Cleaning dist dir \'%s\'' % args.dist_dir) filenames = glob.glob(os.path.join(args.dist_dir, 'subversion-*.tar.gz')) versions = [] for filename in filenames: versions.append(Version(filename)) for k, g in itertools.groupby(sorted(versions), lambda x: (x.major, x.minor)): releases = list(g) logging.info("Saving release '%s'", releases[-1]) for r in releases[:-1]: for filename in glob.glob(os.path.join(args.dist_dir, 'subversion-%s.*' % r)): logging.info("Removing '%s'" % filename) os.remove(filename) #---------------------------------------------------------------------- # Write announcements def write_news(args): 'Write text for the Subversion website.' data = { 'date' : datetime.date.today().strftime('%Y%m%d'), 'date_pres' : datetime.date.today().strftime('%Y-%m-%d'), 'version' : str(args.version), 'version_base' : args.version.base, } if args.version.is_prerelease(): template_filename = 'rc-news.ezt' else: template_filename = 'stable-news.ezt' template = ezt.Template() template.parse(get_tmplfile(template_filename).read()) template.generate(sys.stdout, data) def get_sha1info(args): 'Return a list of sha1 info for the release' sha1s = glob.glob(os.path.join(get_deploydir(args.base_dir), '*.sha1')) class info(object): pass sha1info = [] for s in sha1s: i = info() i.filename = os.path.basename(s)[:-5] i.sha1 = open(s, 'r').read() sha1info.append(i) return sha1info def write_announcement(args): 'Write the release announcement.' sha1info = get_sha1info(args) data = { 'version' : args.version, 'sha1info' : sha1info, 'siginfo' : open('getsigs-output', 'r').read(), 'major-minor' : args.version.base[:3], 'major-minor-patch' : args.version.base, } if args.version.is_prerelease(): template_filename = 'rc-release-ann.ezt' else: template_filename = 'stable-release-ann.ezt' template = ezt.Template(compress_whitespace = False) template.parse(get_tmplfile(template_filename).read()) template.generate(sys.stdout, data) #---------------------------------------------------------------------- # Main entry point for argument parsing and handling def main(): 'Parse arguments, and drive the appropriate subcommand.' # Setup our main parser parser = argparse.ArgumentParser( description='Create an Apache Subversion release.') parser.add_argument('--clean', action='store_true', default=False, help='Remove any directories previously created by %(prog)s') parser.add_argument('--verbose', action='store_true', default=False, help='Increase output verbosity') parser.add_argument('--base-dir', default=os.getcwd(), help='''The directory in which to create needed files and folders. The default is the current working directory.''') subparsers = parser.add_subparsers(title='subcommands') # Setup the parser for the build-env subcommand subparser = subparsers.add_parser('build-env', help='''Download release prerequisistes, including autoconf, libtool, and swig.''') subparser.set_defaults(func=build_env) subparser.add_argument('--sf-mirror', default='softlayer', help='''The mirror to use for downloading files from SourceForge. If in the EU, you may want to use 'kent' for this value.''') subparser.add_argument('--use-existing', action='store_true', default=False, help='''Attempt to use existing build dependencies before downloading and building a private set.''') # Setup the parser for the roll subcommand subparser = subparsers.add_parser('roll', help='''Create the release artifacts.''') subparser.set_defaults(func=roll_tarballs) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') subparser.add_argument('revnum', type=int, help='''The revision number to base the release on.''') subparser.add_argument('--branch', help='''The branch to base the release on.''') # Setup the parser for the post-candidates subcommand subparser = subparsers.add_parser('post-candidates', help='''Build the website to host the candidate tarballs. The default location is somewhere in ~/public_html. ''') subparser.set_defaults(func=post_candidates) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') subparser.add_argument('revnum', type=int, help='''The revision number to base the release on.''') subparser.add_argument('--target', help='''The full path to the destination.''') subparser.add_argument('--code-name', help='''A whimsical name for the release, used only for naming the download directory.''') # The clean-dist subcommand subparser = subparsers.add_parser('clean-dist', help='''Clean the distribution directory (and mirrors) of all but the most recent MAJOR.MINOR release. If no dist-dir is given, this command will assume it is running on people.apache.org.''') subparser.set_defaults(func=clean_dist) subparser.add_argument('--dist-dir', help='''The directory to clean.''') # The write-news subcommand subparser = subparsers.add_parser('write-news', help='''Output to stdout template text for use in the news section of the Subversion website.''') subparser.set_defaults(func=write_news) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') subparser = subparsers.add_parser('write-announcement', help='''Output to stdout template text for the emailed release announcement.''') subparser.set_defaults(func=write_announcement) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') # A meta-target subparser = subparsers.add_parser('clean', help='''The same as the '--clean' switch, but as a separate subcommand.''') subparser.set_defaults(func=cleanup) # Parse the arguments args = parser.parse_args() # first, process any global operations if args.clean: cleanup(args) # Set up logging logger = logging.getLogger() if args.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) # Fix up our path so we can use our installed versions os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \ + os.environ['PATH'] # finally, run the subcommand, and give it the parsed arguments args.func(args) if __name__ == '__main__': main()