diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py new file mode 100755 index 00000000..5b1b066c --- /dev/null +++ b/dev-tools/scripts/buildAndPushRelease.py @@ -0,0 +1,358 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import datetime +import re +import time +import shutil +import os +import sys +import subprocess +import textwrap +import urllib.request, urllib.error, urllib.parse +import xml.etree.ElementTree as ET + +LOG = '/tmp/release.log' + +def log(msg): + f = open(LOG, mode='ab') + f.write(msg.encode('utf-8')) + f.close() + +def run(command): + log('\n\n%s: RUN: %s\n' % (datetime.datetime.now(), command)) + if os.system('%s >> %s 2>&1' % (command, LOG)): + msg = ' FAILED: %s [see log %s]' % (command, LOG) + print(msg) + raise RuntimeError(msg) + +def runAndSendGPGPassword(command, password): + p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE) + f = open(LOG, 'ab') + while True: + p.stdout.flush() + line = p.stdout.readline() + if len(line) == 0: + break + f.write(line) + if line.find(b'Enter GPG keystore password:') != -1: + time.sleep(1.0) + p.stdin.write((password + '\n').encode('UTF-8')) + p.stdin.write('\n'.encode('UTF-8')) + + result = p.poll() + if result != 0: + msg = ' FAILED: %s [see log %s]' % (command, LOG) + print(msg) + raise RuntimeError(msg) + +def load(urlString): + try: + content = urllib.request.urlopen(urlString).read().decode('utf-8') + except Exception as e: + print('Retrying download of url %s after exception: %s' % (urlString, e)) + content = urllib.request.urlopen(urlString).read().decode('utf-8') + return content + +def getGitRev(): + status = os.popen('git status').read().strip() + if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status: + raise RuntimeError('git clone is dirty:\n\n%s' % status) + branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip() + command = 'git log origin/%s..' % branch + unpushedCommits = os.popen(command).read().strip() + if len(unpushedCommits) > 0: + raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, unpushedCommits)) + + print(' git clone is clean') + return os.popen('git rev-parse HEAD').read().strip() + +def prepare(root, version, gpgKeyID, gpgPassword): + print() + print('Prepare release...') + if os.path.exists(LOG): + os.remove(LOG) + + os.chdir(root) + print(' git pull...') + run('git pull') + + rev = getGitRev() + print(' git rev: %s' % rev) + log('\nGIT rev: %s\n' % rev) + + print(' Check DOAP files') + checkDOAPfiles(version) + + print(' ant clean test validate documentation-lint') + run('ant clean test validate documentation-lint') + + open('rev.txt', mode='wb').write(rev.encode('UTF-8')) + + print(' lucene prepare-release') + os.chdir('lucene') + cmd = 'ant -Dversion=%s' % version + if gpgKeyID is not None: + cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID + else: + cmd += ' prepare-release-no-sign' + + if gpgPassword is not None: + runAndSendGPGPassword(cmd, gpgPassword) + else: + run(cmd) + + print(' solr prepare-release') + os.chdir('../solr') + cmd = 'ant -Dversion=%s' % version + if gpgKeyID is not None: + cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID + else: + cmd += ' prepare-release-no-sign' + + if gpgPassword is not None: + runAndSendGPGPassword(cmd, gpgPassword) + else: + run(cmd) + + print(' done!') + print() + return rev + +reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE) +reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<', re.IGNORECASE) +reDoapRevision = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?', re.IGNORECASE) +def checkDOAPfiles(version): + # In Lucene and Solr DOAP files, verify presence of all releases less than the one being produced. + errorMessages = [] + for product in 'lucene', 'solr': + url = 'https://archive.apache.org/dist/lucene/%s' % ('java' if product == 'lucene' else product) + distpage = load(url) + releases = set() + for regex in reVersion1, reVersion2: + for tup in regex.findall(distpage): + if tup[0] in ('1', '2'): # Ignore 1.X and 2.X releases + continue + releases.add(normalizeVersion(tup)) + doapNS = '{http://usefulinc.com/ns/doap#}' + xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS) + doapFile = "dev-tools/doap/%s.rdf" % product + treeRoot = ET.parse(doapFile).getroot() + doapRevisions = set() + for revision in treeRoot.findall(xpathRevision): + match = reDoapRevision.match(revision.text) + if (match is not None): + if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions + doapRevisions.add(normalizeVersion(match.groups())) + else: + errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile)) + missingDoapRevisions = set() + for release in releases: + if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced + missingDoapRevisions.add(release) + if len(missingDoapRevisions) > 0: + errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions)))) + if (len(errorMessages) > 0): + raise RuntimeError('\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)' + % '\n'.join(errorMessages)) + +def normalizeVersion(tup): + suffix = '' + if tup[-1] is not None and tup[-1].lower() == '-alpha': + tup = tup[:(len(tup) - 1)] + suffix = '-ALPHA' + elif tup[-1] is not None and tup[-1].lower() == '-beta': + tup = tup[:(len(tup) - 1)] + suffix = '-BETA' + while tup[-1] in ('', None): + tup = tup[:(len(tup) - 1)] + while len(tup) < 3: + tup = tup + ('0',) + return '.'.join(tup) + suffix + +def pushLocal(version, root, rev, rcNum, localDir): + print('Push local [%s]...' % localDir) + os.makedirs(localDir) + + dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev) + os.makedirs('%s/%s/lucene' % (localDir, dir)) + os.makedirs('%s/%s/solr' % (localDir, dir)) + print(' Lucene') + os.chdir('%s/lucene/dist' % root) + print(' zip...') + if os.path.exists('lucene.tar.bz2'): + os.remove('lucene.tar.bz2') + run('tar cjf lucene.tar.bz2 *') + + os.chdir('%s/%s/lucene' % (localDir, dir)) + print(' unzip...') + run('tar xjf "%s/lucene/dist/lucene.tar.bz2"' % root) + os.remove('%s/lucene/dist/lucene.tar.bz2' % root) + + print(' Solr') + os.chdir('%s/solr/package' % root) + print(' zip...') + if os.path.exists('solr.tar.bz2'): + os.remove('solr.tar.bz2') + run('tar cjf solr.tar.bz2 *') + print(' unzip...') + os.chdir('%s/%s/solr' % (localDir, dir)) + run('tar xjf "%s/solr/package/solr.tar.bz2"' % root) + os.remove('%s/solr/package/solr.tar.bz2' % root) + + print(' KEYS') + run('wget http://home.apache.org/keys/group/lucene.asc') + os.rename('lucene.asc', 'KEYS') + run('chmod a+r-w KEYS') + run('cp KEYS ../lucene') + + print(' chmod...') + os.chdir('..') + run('chmod -R a+rX-w .') + + print(' done!') + return 'file://%s/%s' % (os.path.abspath(localDir), dir) + +def read_version(path): + version_props_file = os.path.join(path, 'lucene', 'version.properties') + return re.search(r'version\.base=(.*)', open(version_props_file).read()).group(1) + +def parse_config(ponymail): + epilogue = textwrap.dedent(''' + Example usage for a Release Manager: + python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1 + ''') + if ponymail: + epilogue = textwrap.dedent(''' + Example usage for a Release Manager: + python3 -u dev-tools/scripts/buildAndPushRelease.py 0.10 + ''') + description = 'Utility to build, push, and test a release.' + parser = argparse.ArgumentParser(description=description, epilog=epilogue, + formatter_class=argparse.RawDescriptionHelpFormatter) + + if ponymail: + parser.add_argument('version', type=str) + return parser.parse_args() + + parser.add_argument('--no-prepare', dest='prepare', default=True, action='store_false', + help='Use the already built release in the provided checkout') + parser.add_argument('--push-local', metavar='PATH', + help='Push the release to the local path') + parser.add_argument('--sign', metavar='KEYID', + help='Sign the release with the given gpg key') + parser.add_argument('--rc-num', metavar='NUM', type=int, default=1, + help='Release Candidate number. Default: 1') + parser.add_argument('--root', metavar='PATH', default='.', + help='Root of Git working tree for lucene-solr. Default: "." (the current directory)') + config = parser.parse_args() + + if not config.prepare and config.sign: + parser.error('Cannot sign already built release') + if config.push_local is not None and os.path.exists(config.push_local): + parser.error('Cannot push to local path that already exists') + if config.rc_num <= 0: + parser.error('Release Candidate number must be a positive integer') + if not os.path.isdir(config.root): + parser.error('Root path "%s" is not a directory' % config.root) + cwd = os.getcwd() + os.chdir(config.root) + config.root = os.getcwd() # Absolutize root dir + if os.system('git rev-parse') or 3 != len([d for d in ('dev-tools','lucene','solr') if os.path.isdir(d)]): + parser.error('Root path "%s" is not a valid lucene-solr checkout' % config.root) + os.chdir(cwd) + + config.version = read_version(config.root) + print('Building version: %s' % config.version) + + if config.sign: + sys.stdout.flush() + import getpass + config.key_id = config.sign + config.key_password = getpass.getpass('Enter GPG keystore password: ') + else: + config.gpg_password = None + + return config + +def check_cmdline_tools(): # Fail fast if there are cmdline tool problems + if os.system('git --version >/dev/null 2>/dev/null'): + raise RuntimeError('"git --version" returned a non-zero exit code.') + antVersion = os.popen('ant -version').read().strip() + if not antVersion.startswith('Apache Ant(TM) version 1.8') and not antVersion.startswith('Apache Ant(TM) version 1.9'): + raise RuntimeError('ant version is not 1.8.X: "%s"' % antVersion) + +def main(): + check_cmdline_tools() + + ponymail = True + c = parse_config(ponymail) + + if ponymail: + + project = 'ponymail' + base_branch_name = 'master' + release_branch_name = c.version + git_archive_format = 'tar.gz' + tarball_name = '~/'+project+'-'+release_branch_name+'.'+git_archive_format + tarball_name_sha256 = tarball_name + '.sha256' + tarball_name_sha256_asc = tarball_name_sha256 + '.asc' + tarball_name_asc = tarball_name + '.asc' + + print('Create a new branch off '+base_branch_name+' called '+release_branch_name+' (ideally, '+base_branch_name+' is always releasable).') + print(' git checkout -b '+release_branch_name+' '+base_branch_name) + + print('Tarball the '+release_branch_name+' branch, sans the .git directory:') + print(' git archive --format='+git_archive_format+' -o '+tarball_name+' HEAD') + + print('Create checksums of the archive (make sure your PGP key is in our KEYS file!):') + + print(' Make a checksum for the archive itself:') + print(' sha256sum '+tarball_name+' > '+tarball_name_sha256) + + print(' Sign the archive:') + print(' gpg --output '+tarball_name_asc+' --sign '+tarball_name) + print(' OR') + print(' gpg --output '+tarball_name_sha256_asc+' --sign '+tarball_name_sha256) + + print('Push the artefacts to https://dist.apache.org/repos/dist/dev/incubator/ponymail/ via subversion') + print(' svn ???') + return + + if c.prepare: + rev = prepare(c.root, c.version, c.key_id, c.key_password) + else: + os.chdir(root) + rev = open('rev.txt', encoding='UTF-8').read() + + if c.push_local: + url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local) + else: + url = None + + if url is not None: + print(' URL: %s' % url) + print('Next run the smoker tester:') + p = re.compile(".*/") + m = p.match(sys.argv[0]) + print('%s -u %ssmokeTestRelease.py %s' % (sys.executable, m.group(), url)) + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print('Keyboard interrupt...exiting') +