#!/usr/bin/env nix-shell #! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests click click-log packaging ])" bundix bundler nix-update nurl curl unzip from __future__ import annotations import click import click_log import shutil import tempfile import re import logging import subprocess import os import stat import json import requests import textwrap from functools import total_ordering from packaging.version import Version from pathlib import Path from typing import Union, Iterable logger = logging.getLogger(__name__) @total_ordering class DiscourseVersion: """Represents a Discourse style version number and git tag. This takes either a tag or version string as input and extrapolates the other. Sorting is implemented to work as expected in regard to A.B.C.betaD version numbers - 2.0.0.beta1 is considered lower than 2.0.0. """ tag: str = "" version: str = "" split_version: Iterable[Union[None, int, str]] = [] def __init__(self, version: str): """Take either a tag or version number, calculate the other.""" if version.startswith('v'): self.tag = version self.version = version.lstrip('v').rstrip("-latest") else: self.tag = 'v' + version self.version = version.rstrip("-latest") self._version = Version(self.version) def __eq__(self, other: DiscourseVersion): """Versions are equal when their individual parts are.""" return self._version == other._version def __gt__(self, other: DiscourseVersion): """Check if this version is greater than the other.""" return self._version > other._version class DiscourseRepo: version_regex = re.compile(r'^v\d+\.\d+\.\d+(\.beta\d+)?$') _latest_commit_sha = None def __init__(self, owner: str = 'discourse', repo: str = 'discourse'): self.owner = owner self.repo = repo @property def versions(self) -> Iterable[str]: r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/git/refs/tags').json() tags = [x['ref'].replace('refs/tags/', '') for x in r] # filter out versions not matching version_regex versions = filter(self.version_regex.match, tags) versions = [DiscourseVersion(x) for x in versions] versions.sort(reverse=True) return versions @property def latest_commit_sha(self) -> str: if self._latest_commit_sha is None: r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/commits?per_page=1') r.raise_for_status() self._latest_commit_sha = r.json()[0]['sha'] return self._latest_commit_sha def get_file(self, filepath, rev): """Return file contents at a given rev. :param str filepath: the path to the file, relative to the repo root :param str rev: the rev to fetch at :return: """ r = requests.get(f'https://raw.githubusercontent.com/{self.owner}/{self.repo}/{rev}/{filepath}') r.raise_for_status() return r.text def _get_build_lock_hash(): nixpkgs_path = Path(__file__).parent / '../../../../' output = subprocess.run(['nix-build', '-A', 'discourse'], text=True, cwd=nixpkgs_path, capture_output=True) # The line is of the form " got: sha256-xxx" lines = [i.strip() for i in output.stderr.splitlines()] new_hash_lines = [i.strip("got:").strip() for i in lines if i.startswith("got:")] if len(new_hash_lines) == 0: if output.returncode != 0: print("Error while fetching new hash with nix build") print(output.stderr) print("No hash change is needed") return None if len(new_hash_lines) > 1: print(new_hash_lines) raise Exception("Got an unexpected number of new hash lines:") return new_hash_lines[0] def _call_nix_update(pkg, version): """Call nix-update from nixpkgs root dir.""" nixpkgs_path = Path(__file__).parent / '../../../../' return subprocess.check_output(['nix-update', pkg, '--version', version], cwd=nixpkgs_path) def _nix_eval(expr: str): nixpkgs_path = Path(__file__).parent / '../../../../' try: output = subprocess.check_output(['nix-instantiate', '--strict', '--json', '--eval', '-E', f'(with import {nixpkgs_path} {{}}; {expr})'], text=True) except subprocess.CalledProcessError: return None return json.loads(output) def _get_current_package_version(pkg: str): return _nix_eval(f'{pkg}.version') def _diff_file(filepath: str, old_version: DiscourseVersion, new_version: DiscourseVersion): repo = DiscourseRepo() current_dir = Path(__file__).parent old = repo.get_file(filepath, old_version.tag) new = repo.get_file(filepath, new_version.tag) if old == new: click.secho(f'{filepath} is unchanged', fg='green') return with tempfile.NamedTemporaryFile(mode='w') as o, tempfile.NamedTemporaryFile(mode='w') as n: o.write(old), n.write(new) o.flush(), n.flush() width = shutil.get_terminal_size((80, 20)).columns diff_proc = subprocess.run( ['diff', '--color=always', f'--width={width}', '-y', o.name, n.name], stdout=subprocess.PIPE, cwd=current_dir, text=True ) click.secho(f'Diff for {filepath} ({old_version.version} -> {new_version.version}):', fg='bright_blue', bold=True) click.echo(diff_proc.stdout + '\n') return def _remove_platforms(rubyenv_dir: Path): for platform in ['arm64-darwin-20', 'x86_64-darwin-18', 'x86_64-darwin-19', 'x86_64-darwin-20', 'x86_64-linux', 'aarch64-linux']: with open(rubyenv_dir / 'Gemfile.lock', 'r') as f: for line in f: if platform in line: subprocess.check_output( ['bundle', 'lock', '--remove-platform', platform], cwd=rubyenv_dir) break @click_log.simple_verbosity_option(logger) @click.group() def cli(): pass @cli.command() @click.argument('rev', default='latest') @click.option('--reverse/--no-reverse', default=False, help='Print diffs from REV to current.') def print_diffs(rev, reverse): """Print out diffs for files used as templates for the NixOS module. The current package version found in the nixpkgs worktree the script is run from will be used to download the "from" file and REV used to download the "to" file for the diff, unless the '--reverse' flag is specified. REV should be the git rev to find changes in ('vX.Y.Z') or 'latest'; defaults to 'latest'. """ if rev == 'latest': repo = DiscourseRepo() rev = repo.versions[0].tag old_version = DiscourseVersion(_get_current_package_version('discourse')) new_version = DiscourseVersion(rev) if reverse: old_version, new_version = new_version, old_version for f in ['config/nginx.sample.conf', 'config/discourse_defaults.conf']: _diff_file(f, old_version, new_version) @cli.command() @click.argument('rev', default='latest') def update(rev): """Update gem files and version. REV: the git rev to update to ('vX.Y.Z[.betaA]') or 'latest'; defaults to 'latest'. """ repo = DiscourseRepo() if rev == 'latest': version = repo.versions[0] else: version = DiscourseVersion(rev) logger.debug(f"Using rev {version.tag}") logger.debug(f"Using version {version.version}") rubyenv_dir = Path(__file__).parent / "rubyEnv" for fn in ['Gemfile.lock', 'Gemfile']: with open(rubyenv_dir / fn, 'w') as f: f.write(repo.get_file(fn, version.tag)) # download and copy the gems that are specified as path dependencies so bundle/bundix works # currently this is only `migrations` with tempfile.TemporaryDirectory() as d: print("downloading discourse gems (path dependencies) for bundix") subprocess.check_output(['curl',f'https://codeload.github.com/discourse/discourse/zip/refs/tags/{version.tag}','-o','discourse.zip'], cwd=d) subprocess.check_output(['unzip', f'discourse.zip'], cwd=d) subprocess.check_output(['mv', f'discourse-{version.version}/migrations', rubyenv_dir / 'migrations'], cwd=d) # -- sed commands borrowed from ../../../by-name/gi/gitlab/update.py which has a similar predicament with path deps # comment from that file: "Undo our gemset.nix patches so that bundix runs through" subprocess.check_output( ["sed", "-i", "-e", "s|\\${src}/||g", "gemset.nix"], cwd=rubyenv_dir ) subprocess.check_output( ["sed", "-i", "-e", "s|^src:[[:space:]]||g", "gemset.nix"], cwd=rubyenv_dir ) # work around https://github.com/nix-community/bundix/issues/8 os.environ["BUNDLE_FORCE_RUBY_PLATFORM"] = "true" subprocess.check_output(['bundle', 'lock'], cwd=rubyenv_dir) _remove_platforms(rubyenv_dir) subprocess.check_output(['bundix'], cwd=rubyenv_dir) # -- also from gitlab pkg, see above subprocess.check_output( [ "sed", "-i", "-e", "1c\\src: {", "-e", 's:path = \\(migrations/[^;]*\\);:path = "${src}/\\1";:g', "gemset.nix", ], cwd=rubyenv_dir, ) subprocess.check_output(["rm", "-rf", "migrations"], cwd=rubyenv_dir) # -- end gitlab pkg code # update the sass-embedded override # must run *after* the gemfile update! dart_sass_ver = _nix_eval('discourse.rubyEnv.gemset.sass-embedded.version') for platform in ["x64", "arm64"]: prev_hash = _nix_eval(f'discourse.rubyEnv.dart-{platform}-hash') new_hash = subprocess.check_output([ "nurl", "--fetcher", "fetchzip", "--hash", f"https://github.com/sass/dart-sass/releases/download/{dart_sass_ver}/dart-sass-{dart_sass_ver}-linux-{platform}.tar.gz", ], text=True).strip("\n") if prev_hash == new_hash: click.echo(f"dart-sass for {platform} up to date!") continue click.echo(f"Update vendored dart-sass for {platform} to {dart_sass_ver} (hash {prev_hash} -> {new_hash})") with open(Path(__file__).parent / "default.nix", 'r+') as f: content = f.read() content = content.replace(prev_hash, new_hash) f.seek(0) f.write(content) f.truncate() # update the discourse package itself _call_nix_update('discourse', version.version) old_pnpm_hash = _nix_eval('discourse.assets.pnpmDeps.outputHash') new_pnpm_hash = _get_build_lock_hash() if new_pnpm_hash is not None: click.echo(f"Updating pnpm lock hash: {old_pnpm_hash} -> {new_pnpm_hash}") with open(Path(__file__).parent / "default.nix", 'r+') as f: content = f.read() content = content.replace(old_pnpm_hash, new_pnpm_hash) f.seek(0) f.write(content) f.truncate() @cli.command() @click.argument('rev', default='latest') def update_mail_receiver(rev): """Update discourse-mail-receiver. REV: the git rev to update to ('vX.Y.Z') or 'latest'; defaults to 'latest'. """ repo = DiscourseRepo(repo="mail-receiver") if rev == 'latest': version = repo.versions[0] else: version = DiscourseVersion(rev) _call_nix_update('discourse-mail-receiver', version.version) @cli.command() def update_plugins(): """Update plugins to their latest revision.""" plugins = [ {'name': 'discourse-bbcode-color'}, {'name': 'discourse-docs'}, {'name': 'discourse-events', 'owner': 'angusmcleod'}, {'name': 'discourse-ldap-auth', 'owner': 'jonmbake'}, {'name': 'discourse-prometheus'}, {'name': 'discourse-saved-searches'}, {'name': 'discourse-yearly-review'}, ] for plugin in plugins: fetcher = plugin.get('fetcher') or "fetchFromGitHub" owner = plugin.get('owner') or "discourse" name = plugin.get('name') repo_name = plugin.get('repo_name') or name if fetcher == "fetchFromGitHub": url = f"https://github.com/{owner}/{repo_name}" else: raise NotImplementedError(f"Missing URL pattern for {fetcher}") repo = DiscourseRepo(owner=owner, repo=repo_name) # implement the plugin pinning algorithm laid out here: # https://meta.discourse.org/t/pinning-plugin-and-theme-versions-for-older-discourse-installs/156971 # this makes sure we don't upgrade plugins to revisions that # are incompatible with the packaged Discourse version repo_latest_commit = repo.latest_commit_sha try: compatibility_spec = repo.get_file('.discourse-compatibility', repo_latest_commit) versions = [(DiscourseVersion(discourse_version), plugin_rev.strip(' ')) for [discourse_version, plugin_rev] in [line.lstrip("< ").split(':') for line in compatibility_spec.splitlines() if line != '']] discourse_version = DiscourseVersion(_get_current_package_version('discourse')) versions = list(filter(lambda ver: ver[0] >= discourse_version, versions)) if versions == []: rev = repo_latest_commit else: rev = versions[0][1] print(rev) except requests.exceptions.HTTPError: rev = repo_latest_commit filename = _nix_eval(f'builtins.unsafeGetAttrPos "src" discourse.plugins.{name}') if filename is None: filename = Path(__file__).parent / 'plugins' / name / 'default.nix' filename.parent.mkdir() has_ruby_deps = False for line in repo.get_file('plugin.rb', rev).splitlines(): if 'gem ' in line: has_ruby_deps = True break with open(filename, 'w') as f: f.write(textwrap.dedent(f""" {{ lib, mkDiscoursePlugin, fetchFromGitHub }}: mkDiscoursePlugin {{ name = "{name}";"""[1:] + (""" bundlerEnvArgs.gemdir = ./.;""" if has_ruby_deps else "") + f""" src = {fetcher} {{ owner = "{owner}"; repo = "{repo_name}"; rev = "replace-with-git-rev"; sha256 = "replace-with-sha256"; }}; meta = {{ homepage = ""; maintainers = with lib.maintainers; [ ]; license = lib.licenses.mit; # change to the correct license! description = ""; }}; }}""")) all_plugins_filename = Path(__file__).parent / 'plugins' / 'all-plugins.nix' with open(all_plugins_filename, 'r+') as f: content = f.read() pos = -1 while content[pos] != '}': pos -= 1 content = content[:pos] + f' {name} = callPackage ./{name} {{}};' + os.linesep + content[pos:] f.seek(0) f.write(content) f.truncate() else: filename = filename['file'] prev_commit_sha = _nix_eval(f'discourse.plugins.{name}.src.rev') if prev_commit_sha == rev: click.echo(f'Plugin {name} is already at the latest revision') continue prev_hash = _nix_eval(f'discourse.plugins.{name}.src.outputHash') new_hash = subprocess.check_output([ "nurl", "--fetcher", fetcher, "--hash", url, rev, ], text=True).strip("\n") click.echo(f"Update {name}, {prev_commit_sha} -> {rev} in {filename}") with open(filename, 'r+') as f: content = f.read() content = content.replace(prev_commit_sha, rev) content = content.replace(prev_hash, new_hash) f.seek(0) f.write(content) f.truncate() rubyenv_dir = Path(filename).parent gemfile = rubyenv_dir / "Gemfile" version_file_regex = re.compile(r'.*File\.expand_path\("\.\./(.*)", __FILE__\)') gemfile_text = '' plugin_file = repo.get_file('plugin.rb', rev) plugin_file = plugin_file.replace(",\n", ", ") # fix split lines for line in plugin_file.splitlines(): if 'gem ' in line: line = ','.join(filter(lambda x: ":require_name" not in x and "require_name:" not in x, line.split(','))) gemfile_text = gemfile_text + line + os.linesep version_file_match = version_file_regex.match(line) if version_file_match is not None: filename = version_file_match.groups()[0] content = repo.get_file(filename, rev) with open(rubyenv_dir / filename, 'w') as f: f.write(content) if len(gemfile_text) > 0: if os.path.isfile(gemfile): os.remove(gemfile) subprocess.check_output(['bundle', 'init'], cwd=rubyenv_dir) os.chmod(gemfile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH) with open(gemfile, 'a') as f: f.write(gemfile_text) subprocess.check_output(['bundle', 'lock', '--add-platform', 'ruby'], cwd=rubyenv_dir) subprocess.check_output(['bundle', 'lock', '--update'], cwd=rubyenv_dir) _remove_platforms(rubyenv_dir) subprocess.check_output(['bundix'], cwd=rubyenv_dir) if __name__ == '__main__': cli()