#!/usr/bin/env python """ Attempt to fixup rpm specfiles of Python 2 modules for Python 3 Caveats: - will require a human eye to ensure the results are sane - it doesn't attempt to change the source URL; does upstream release separate 2 vs 3 tarballs? """ import sys import re import os class BadExitCode(Exception): def __init__(self, invocation): self.invocation = invocation def __str__(self): result = '''Unexpected exit code from invocation of "%s": %s (expecting %s) --- stderr --- %s --- end of stderr ''' % (self.invocation.args, self.invocation.returncode, self.invocation.expectedExitCode, self.invocation.stderr) return result class Invocation(object): def __init__(self, args, expectedExitCode=0, logger=None, capture_stdout=True, capture_stderr=True, env=None): from subprocess import Popen, PIPE self.args = args if logger: logger('Invoking %s' % args) self.expectedExitCode = expectedExitCode kwargs = dict(env=env) if capture_stdout: kwargs['stdout'] = PIPE if capture_stderr: kwargs['stderr'] = PIPE self.popen = Popen(args, **kwargs) self.stdout,self.stderr = self.popen.communicate() self.returncode = self.popen.returncode # use None to avoid caring about the exit code: if expectedExitCode is not None: if self.returncode!=expectedExitCode: raise BadExitCode(self) if logger: if self.stdout: for line in self.stdout.splitlines(): logger(line) if self.stderr: for line in self.stderr.splitlines(): logger(line) def invoke(args, expectedExitCode=0, logger=None, capture_stdout=True, capture_stderr=True, env=None): return Invocation(args, expectedExitCode, logger, capture_stdout, capture_stderr, env) def get_specfile_name(srcname): if srcname.startswith('python-'): return srcname.replace('python-', 'python3-') return 'python3-%s' % srcname def get_dst_path(srcpath): (path, filename) = os.path.split(srcpath) # print (path, filename) return os.path.join(path, get_specfile_name(filename)) def build_rpm(specfile): path, filename = os.path.split(specfile) path = os.path.abspath(path) extradefines = ['--define', '_sourcedir %s' % path, '--define', '_specdir %s' % path, '--define', '_builddir %s' % path, '--define', '_srcrpmdir %s' % path, '--define', '_rpmdir %s' % path] #$(DIST_DEFINES) args = ['rpmbuild', '-ba' ] + extradefines + [specfile] invoke(args) def build_srpm(specfile, logger = None): path, filename = os.path.split(specfile) path = os.path.abspath(path) extradefines = ['--define', '_sourcedir %s' % path, '--define', '_specdir %s' % path, '--define', '_builddir %s' % path, '--define', '_srcrpmdir %s' % path, '--define', '_rpmdir %s' % path] #$(DIST_DEFINES) args = ['rpmbuild', '-bs' ] + extradefines + [os.path.abspath(specfile)] invoke(args, logger=logger) class BuildRequirement(object): def __init__(self, name, conditional=None, arg=None): self.name = name self.conditional = conditional self.arg = arg def __str__(self): if self.conditional: return "%s %s %s" % (self.name, self.conditional, self.arg) else: return self.name def as_3(self): return BuildRequirement(get_specfile_name(self.name), self.conditional, self.arg) @classmethod def from_spec_line(cls, line): # Get a list of BuildRequirement instances: result = [] m = re.match('^BuildRequires:(.*)', line) if m: for commagroup in m.group(1).split(','): spacegroup = commagroup.split() if len(spacegroup) == 3 and spacegroup[1] in ['<', '<=', '=', '>', '>=']: result += [BuildRequirement(*spacegroup)] else: result += [BuildRequirement(item, None, None) for item in spacegroup] return result class SpecfileConversion(object): def __init__(self, srcpath, dstpath = None): self.srcpath = srcpath self.srcname = os.path.split(srcpath)[1][:-5] # lose the trailing ".spec" self.dstname = get_specfile_name(self.srcname) if dstpath is None: dstpath = get_dst_path(srcpath) self.dstpath = dstpath self.buildrequires = [] self.specfile = '' self._output = open(self.dstpath, 'w') self._write('# We need python3-tools for /usr/bin/2to3-3\nBuildRequires: python3-tools\n') # FIXME state = None for line in open(srcpath): if line.startswith('%'): # Handle switching to new sections within the spec: newstate = line[1:].strip() if newstate in ['prep', 'build', 'install', 'check', 'files', 'changelog']: state = newstate if newstate == 'build': # End of %prep section: if True: #self.srcname not in ['python-setuptools']: self._write("# Invoke python3's 2to3 on the prepped source tree:\n") if False: # in case the %prep code has done a cd; this is actually one level too high: self._write('cd %{_sourcedir}\n') # Invoke with --no-diffs: I've seen it fail (on docutils-0.5.tar.gz) with encoding # issues trying to write the diffs as ASCII: self._write('2to3-3 --no-diffs --write .\n') self._write('\n') # FIXME: for now, throw away all %check sections if False: if state == 'check': continue self.buildrequires += BuildRequirement.from_spec_line(line) if state != 'changelog': # Fix up "Name:" line: m = re.match('(Name:\s*)(\S+)(.*)', line) if m: line = m.group(1) + self.dstname + m.group(3) + '\n' # python3-devel defines these macros: if (line.startswith('%{!?python_sitelib') or line.startswith('%{!?python_sitearch')): continue line = line.replace('import sys ; print sys.version[:3]', 'import sys ; sys.stdout.write(sys.version[:3])') if not(line.startswith('Source') or line.startswith('Patch')): line = line.replace('python-', 'python3-') line = line.replace('__python', '__python3') line = line.replace('python_sitelib', 'python3_sitelib') line = line.replace('python_sitearch', 'python3_sitearch') line = line.replace('print get_python_lib()', 'import sys; sys.stdout.write(get_python_lib())') line = line.replace('print get_python_lib(1)', 'import sys; sys.stdout.write(get_python_lib(1))') if not line.startswith('License:'): # FIXME: not "Python 3 Enhancement" for PEP references line = line.replace('Python ', 'Python 3 ') # Eliminate old "Obsoletes" lines; we're building a new stack, don't # want to risk name collisions: if line.startswith('Obsoletes:'): continue if line.startswith('Source') or line.startswith('Patch'): line = line.replace('%{name}', self.srcname) # Just invoke setup.py directly, rather than use execfile, which # doesn't exist in py3k: line = line.replace("-c 'import setuptools; execfile(\"setup.py\")'", 'setup.py') self._write(line) self._output.close() def _write(self, line): assert line.endswith('\n') #print line, self._output.write(line) self.specfile += line class RecursiveBuild(object): def __init__(self, mock_config, workdir, srcname): self.mock_config = mock_config self.srcname = srcname self.workdir= os.path.abspath(workdir) self.builtrpms = ['python', 'libtool', 'doxygen', 'libxslt'] + list(self.get_repo_rpms()) self.recursive_build(srcname, 0) def log(self, s, depth): print ' ' * depth, s def get_chroot_rpms(self): for line in invoke(['mock', '-v', '-r', self.mock_config, '--shell', 'rpm -qa']).stdout.splitlines(): m = re.match('(.+)-(.+)-(.+)', line) if m: yield m.group(1) def get_repo_rpms(self): for file in os.listdir(os.path.join(self.workdir, 'BUILT_RPMS')): m = re.match('(.+)-(.+)-(.+)', file) if m: yield m.group(1) def get_chroot_rpm_names(self): return [name for (name, version, release) in self.get_chroot_rpms()] def invoke(self, args, depth, capture_stdout=True, capture_stderr = True, env=None): invoke(args, logger=lambda s: self.log(s, depth), capture_stdout=capture_stdout, capture_stderr=capture_stderr, env=env) def chworkdir(self): os.chdir(os.path.abspath(self.workdir)) def recursive_build(self, srcname, depth=0): if not os.path.exists(os.path.abspath(self.workdir)): os.mkdir(os.path.abspath(self.workdir)) self.chworkdir() srcdir = os.path.join(srcname, 'devel') if not os.path.exists(srcdir): self.invoke(['cvs', 'co', srcname], depth, env={'CVSROOT':':pserver:anonymous@cvs.fedoraproject.org:/cvs/pkgs'}) os.chdir(srcdir) for line in open('sources'): sum, file = line.split() if not os.path.exists(file): self.invoke(['make', 'sources'], depth) break self.chworkdir() os.chdir(srcdir) srcpath = '%s.spec' % srcname c = SpecfileConversion(srcpath) self.log('Building %s from %s' % (c.dstname, srcname), depth) if False: print '%s%s:\n2: %s\n3: %s ' % (' '*depth, c.dstname, ','.join([str(br) for br in c.buildrequires]), ','.join([str(br.as_3()) for br in c.buildrequires])) # Create SRPM: self.log('Creating SRPM for %s' % c.dstname, depth + 1) self.chworkdir() os.chdir(srcdir) build_srpm(c.dstpath, logger=lambda s: self.log(s, depth + 1)) # Recurse, building the build requirements: self.log('Checking build requirements for %s' % c.dstname, depth + 1) for br in c.buildrequires: br3 = br.as_3() if br.name != br3.name: # ... then this is a python3- specific name. # If we need "foo-devel", we probably need to build "foo": depname = br.name.replace('-devel', '') if depname == 'python-elementtree': continue if br3.name == 'python3-setuptools-devel': br3.name = 'python3-setuptools' if (depname not in self.builtrpms and br3.name not in self.builtrpms #and depname not in self.get_chroot_rpm_names() ): self.log('Need %s from %s for %s (builtrpms: %s)' % (br3.name, depname, c.dstname, ','.join(self.builtrpms)), depth+2) self.recursive_build(depname, depth+3) # Create the RPM: self.log('Building RPM for %s' % c.dstname, depth+1) self.chworkdir() os.chdir(srcdir) src_rpm_file = None for file in os.listdir('.'): if file.startswith('%s-' % c.dstname): src_rpm_file = file break sys.stdout.flush() self.chworkdir() self.invoke(['mock', '-v', '-r', self.mock_config, '--resultdir=%s' % os.path.abspath('BUILT_RPMS'), '--rebuild', os.path.join(srcdir, src_rpm_file)], depth, capture_stdout = False, capture_stderr = False) # Regenerate the repo, with the newly-built rpms: self.log('Regenerating repo', depth+1) self.invoke(['createrepo', 'BUILT_RPMS'], depth+2) # Note that we've now built this rpm: self.builtrpms.append(srcname) # mkdir builtrpms # cp ~david/rpmbuild/RPMS/i686/*-3.1.1-8* builtrpms # createrepo builtrpms/ def main(): workdir = os.path.abspath(os.path.join(os.getcwd(), 'workdir')) srcname = sys.argv[1] if srcname.endswith('.spec'): c = SpecfileConversion(srcname) else: rb = RecursiveBuild('fedora-11-i386-with-my-repo', workdir, srcname) if __name__ == '__main__': main()