#!/usr/bin/python3 import click import json import re import datetime import os import shutil import email.utils import subprocess import requests import base64 from OpenSSL import crypto def parse_version(s): return tuple([int(i) if i.isdigit() else i for i in re.sub('([0-9]+)', ' \\1 ', s).split()]) def is_stable_release(release): if release['isNightly']: return False for token in ('alpha', 'beta', 'rc'): if token in release['version'].lower(): return False return True def download_and_unpack(app): release = app['releases'][0] url = release['download'] shutil.rmtree('src', ignore_errors=True) os.mkdir('src') store = crypto.X509Store() with open('data/root.crl', 'r') as f: crl = crypto.load_crl(crypto.FILETYPE_PEM, f.read()) store.add_crl(crl) with open('config/root.crt', 'r') as f: root_cert_chain = f.read() for cert_data in root_cert_chain.split('-----BEGIN CERTIFICATE-----')[1:]: cert_data = '-----BEGIN CERTIFICATE-----' + cert_data cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_data) store.add_cert(cert) store.set_flags(crypto.X509StoreFlags.CRL_CHECK) cert = crypto.load_certificate(crypto.FILETYPE_PEM, app['certificate']) store_ctx = crypto.X509StoreContext(store, cert) store_ctx.verify_certificate() assert cert.get_subject().CN == app['id'] print('Downloading app archive') r = requests.get(url) print('Verifiying signature') result = crypto.verify(cert, base64.b64decode(release['signature']), r.content, release['signatureDigest']) assert result is None print('Unpacking archive') unpack_proc = subprocess.Popen(['tar', '-C', 'src', '-xz'], stdin=subprocess.PIPE) unpack_proc.stdin.write(r.content) unpack_proc.stdin.close() if unpack_proc.wait(): raise Exception(f'Unpacking "{{ url }}" failed') def get_debian_packages_for_php_extension(name, minver=None, maxver=None): orig_name = name name = name.lower() if name == 'libxml': # libxml is builtin, but some apps have version requirements for the underlying libxml2 res = [] if minver is not None: res.append(f'libxml2 (>= {minver})') if maxver is not None: res.append(f'libxml2 (<< {maxver})') return res if name in {'calendar', 'ctype', 'exif', 'ffi', 'fileinfo', 'ftp', 'gettext', 'iconv', 'pdo', 'phar', 'posix', 'shmop', 'sockets', 'sysvmsg', 'sysvsem', 'sysvshm', 'tokenizer'}: assert minver is None and maxver is None # version constraint by php version return [] # php-common is already depended on by php if name in {'dom', 'simplexml', 'xml', 'xmlreader', 'xmlwriter', 'xsl'}: assert minver is None and maxver is None # version constraint by php version return ['php-xml'] # Standard extensions that are not included in "php" if name in {'fpm', 'phpdbg', 'xsl', 'bcmath', 'bz2', 'curl', 'dba', 'enchant', 'gd', 'gmp', 'imap', 'interbase', 'intl', 'json', 'ldap', 'mbstring', 'mysql', 'odbc', 'opcache', 'pgsql', 'pspell', 'readline', 'snmp', 'soap', 'sqlite3', 'sybase', 'tidy', 'xmlrpc', 'zip'}: assert minver is None and maxver is None # version constraint by php version return [f'php-{ name }'] raise Exception(f'Unknown PHP extension: {orig_name}') def parse_version_spec(expr): if expr == '*': return None, None match = re.fullmatch(r'(?:>=([0-9]+\.[0-9]+\.[0-9]+))? ?(?:<([0-9]+\.[0-9]+\.[0-9]+))?', expr) if not match: raise Exception(f'Invalid version specification: {expr}') return match.groups() def create_debian_dir(app): shutil.rmtree('debian', ignore_errors=True) shutil.copytree('debian_static', 'debian') release = app['releases'][0] dependencies = [] minver, maxver = parse_version_spec(release['platformVersionSpec']) if minver is not None: dependencies.append(f'nextcloud (>= {minver})') if maxver is not None: dependencies.append(f'nextcloud (<< {maxver})') if minver is None and maxver is None: dependencies.append(f'nextcloud') minver, maxver = parse_version_spec(release['phpVersionSpec']) if minver is not None: minver = re.sub(r'\.[0-9]+$', '', minver) # package version only has two digits dependencies.append(f'php (>= 2:{minver})') if maxver is not None: maxver = re.sub(r'\.[0-9]+$', '', maxver) # package version only has two digits dependencies.append(f'php (<< 2:{maxver})') if minver is None and maxver is None: dependencies.append(f'php') for php_ext in release['phpExtensions']: minver, maxver = parse_version_spec(php_ext['versionSpec']) for dep in get_debian_packages_for_php_extension(php_ext['id'], minver, maxver): if dep not in dependencies: dependencies.append(dep) with open('debian/control', 'w') as f: f.write(f'''\ Source: nextcloud-app-{ app['id'].replace('_', '-') } Section: web Priority: optional Maintainer: CCCV <it@cccv.de> Build-Depends: debhelper-compat (= 12) Standards-Version: 4.5.0 Homepage: { app['website'] } Package: nextcloud-app-{ app['id'].replace('_', '-') } Architecture: all Depends: { ', '.join(dependencies) } Description: Nextcloud { app['id'] } app ''') with open('debian/changelog', 'w') as f: release_date = datetime.datetime.fromisoformat(release['created'].rstrip('Z')) f.write(f'''\ nextcloud-app-{ app['id'].replace('_', '-') } ({ release['version'] }) unstable; urgency=medium Release { release['version'] } -- autoupdater <infra+packages-autoupdate@cccv.de> { email.utils.formatdate(release_date.timestamp()) } ''') prev_date = release_date for release in app['releases'][1:]: release_date = datetime.datetime.fromisoformat(release['created'].rstrip('Z')) if release_date > prev_date: continue prev_date = datetime.datetime.fromisoformat(release['created'].rstrip('Z')) f.write(f'''\ nextcloud-app-{ app['id'].replace('_', '-') } ({ release['version'] }) unstable; urgency=medium Release { release['version'] } -- autoupdater <infra+packages-autoupdate@cccv.de> { email.utils.formatdate(release_date.timestamp()) } ''') with open('debian/install', 'w') as f: f.write(f'''\ src/{ app['id'] } /usr/share/nextcloud/apps/ ''') @click.group() def cli(): pass @cli.command() @click.argument('target') def prepare(target): app_id, version, *_ = target.split('/', 2) + [None] with open('data/apps.json') as f: apps = json.load(f) app = [item for item in apps if item['id'] == app_id][0] app['releases'] = [release for release in app['releases'] if is_stable_release(release)] if version is not None: app['releases'] = [release for release in app['releases'] if parse_version(release['version']) <= parse_version(version)] download_and_unpack(app) create_debian_dir(app) @cli.command() def update(): #r = requests.get('https://apps.nextcloud.com/api/v1/platforms.json') #platforms = r.json() #platforms.sort(key=lambda platform: parse_version(platform['version'])) #platforms = [platform for platform in platforms if platform['hasRelease']] #latest_stable_version = platforms[-1]['version'] #if latest_stable_version.endswith('.0.0'): # latest_stable_version = platforms[-2]['version'] latest_stable_version = '29.0.5' r = requests.get(f'https://apps.nextcloud.com/api/v1/platform/{latest_stable_version}/apps.json') apps = r.json() apps.sort(key=lambda app: app['id']) for app in apps: # Sort releases semantically instead of alphanumerically (i.e. 1.0.10 before 1.0.2) app['releases'].sort(key=lambda release: parse_version(release['version']), reverse=True) with open('data/apps.json', 'w') as f: json.dump(apps, f, indent=2, ensure_ascii=False) r = requests.get('https://raw.githubusercontent.com/nextcloud/server/master/resources/codesigning/root.crl') with open('data/root.crl', 'wb') as f: f.write(r.content) @cli.command() def get_apps(): with open('data/apps.json') as f: apps = json.load(f) for app in apps: print(app['id']) @cli.command() @click.argument('app_id') def get_releases(app_id): with open('data/apps.json') as f: apps = json.load(f) app = [item for item in apps if item['id'] == app_id][0] for release in app['releases']: if is_stable_release(release): print(release['version']) if __name__ == '__main__': cli()