From 83ace5827fb99ddcfc433fa9bd4e97d403e20476 Mon Sep 17 00:00:00 2001 From: Vladimir Borisov Date: Wed, 17 Apr 2024 14:13:55 +0300 Subject: [PATCH] Generate auto update RC descriptor --- .github/workflows/build.yml | 4 +- generate_descriptor.js | 299 ++++++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 generate_descriptor.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98063ed..3380d7f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,9 @@ jobs: - name: Upload to Amazon S3 run: | aws s3 cp --acl public-read ".\StremioSetup-v$((get-item .\StremioSetup*.exe).VersionInfo.ProductVersion.Trim()).exe" s3://stremio-artifacts/stremio-shell-ng/${{ github.ref_name }}/ - + - name: Generate RC descriptor + run: | + node ./generate_descriptor.js --wait-all --tag=${{ github.ref_name }} - uses: actions/upload-artifact@v3 with: name: stremio-setup diff --git a/generate_descriptor.js b/generate_descriptor.js new file mode 100644 index 0000000..10ea3ea --- /dev/null +++ b/generate_descriptor.js @@ -0,0 +1,299 @@ +#!/usr/bin/env node + +// Copyright (C) 2017-2024 Smart Code OOD 203358507 + +// This script generates an auto update descriptor for a given tag. +// It will get the files for the tag, calculate their hashes, and upload +// the descriptor to S3. In order to run it, you need to have aws cli installed +// and configured with your AWS credentials. + +const { exec } = require("child_process"); +const { basename, posix } = require("path"); +const https = require("https"); +const { createHash } = require("crypto"); +const { tmpdir } = require("os"); +const fs = require("fs"); +const S3_BUCKET_PATH = "s3://stremio-artifacts/stremio-shell-ng/"; +const S3_VERSIONS_PATH = "s3://stremio-artifacts/stremio-shell-ng/[versions]/"; +const S3_VERSIONS_RC_PATH = + "s3://stremio-artifacts/stremio-shell-ng/[versions]/rc/"; +const DOWNLOAD_ENDPOINT = "https://dl.strem.io/stremio-shell-ng/"; +const OS_EXT = { + ".exe": "windows", +}; +const VERSION_REGEX = /^v(\d+\.\d+\.\d+).*$/; + +const supportedArguments = Object.freeze({ + tag: { + description: + "The tag to generate the descriptor for. If not specified, the latest tag will be used. A tag must start with a v followed by the version number in semver format, followed by an optional suffix. For example: v1.2.3, v1.2.3-rc.1, v1.2.3-beta.2", + parse: parseTagArgument, + }, + force: { + description: "Overwrite the descriptor if it already exists.", + default: false, + parse: parseBooleanArgument, + }, + dry_run: { + description: + "Do not upload the descriptor to S3. Just print it to stdout. The descriptor is printed even if --quiet is set.", + default: false, + parse: parseBooleanArgument, + }, + wait_all: { + description: + "By default at least one file is required to produce a descriptor. This flag will cause the script to exit (without error) if not all files are uploaded before generating the descriptor.", + default: false, + parse: parseBooleanArgument, + }, + release: { + description: + "If this flag is set, the descriptor will be uploaded to the release path instead of the release candidate path.", + default: false, + parse: parseBooleanArgument, + }, + quiet: { + description: "Suppress all output except for errors.", + default: false, + parse: parseBooleanArgument, + }, + help: { + description: "Print this help message", + parse: () => { + usage(); + process.exit(0); + }, + }, +}); + +function parseBooleanArgument(value) { + // Acceptable falsy values. Note that an empty string is considered truthy. + // Thus --option and --option= will set option to true. + return ["false", "0", "no", "off"].includes(value.toLowerCase()) + ? false + : true; +} + +function parseTagArgument(value) { + if (value.match(VERSION_REGEX) !== null) return value; +} + +function usage() { + log(`Usage: ${basename(process.argv[1])} [options]`); + log("Options:"); + Object.keys(supportedArguments).forEach((key) => { + log( + ` --${key.replace(/_/g, "-")}${typeof supportedArguments[key].default !== "undefined" + ? " [default: " + supportedArguments[key].default.toString() + "]" + : "" + }` + ); + log(` ${supportedArguments[key].description}`); + }); +} + +const parseArguments = () => { + const args = Object.keys(supportedArguments).reduce((acc, key) => { + if (typeof supportedArguments[key].default !== "undefined") + acc[key] = supportedArguments[key].default; + return acc; + }, {}); + try { + for (let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i]; + if (arg.startsWith("--")) { + // Stop processing arguments after -- + if (arg.length === 2) break; + const eq_position = arg.indexOf("="); + const name_end = eq_position === -1 ? arg.length : eq_position; + const name = arg.slice(2, name_end).replace(/-/g, "_"); + if (!supportedArguments[name]) + throw new Error(`Unsupported argument ${arg}`); + const value = supportedArguments[name].parse(arg.slice(name_end + 1)); + if (typeof value === "undefined") + throw new Error( + `Invalid value for argument --${name.replace(/_/g, "-")}` + ); + args[name] = value; + } + } + } catch (e) { + console.error(e.message); + usage(); + process.exit(1); + } + return args; +}; + +const args = parseArguments(); + +function log(...params) { + if (!args.quiet) console.info(...params); +} + +const s3Cmd = (command_line) => { + return new Promise((resolve, reject) => { + exec(`aws s3 ${command_line}`, (err, stdout, stderr) => { + const error = err || (stderr && new Error(stderr)); + if (error) return reject(error); + resolve(stdout); + }); + }); +}; + +const s3Ls = (path) => s3Cmd(`ls --no-paginate ${path}`).catch(() => { }); +const s3Cp = (src, dest) => s3Cmd(`cp --acl public-read ${src} ${dest}`); + +// Downloads a file from S3 and returns a hash of it +const s3Hash = (path) => { + return new Promise((resolve, reject) => { + const hash = createHash("sha256"); + https + .get(path, (res) => { + res.on("data", hash.update.bind(hash)); + res.on("end", () => { + resolve(hash.digest("hex")); + }); + }) + .on("error", reject); + }); +}; + +// Parses a line from the S3 listing +const parseS3Listing = (tag) => (line) => { + const path = ( + line.match( + /^\s*(?\d{4}(?:-\d{2}){2} \d\d(?::\d\d){2})\s+\d+\s+(?.*)\s*$/ + ) || {} + ).groups; + if (!path) return; + const os = OS_EXT[posix.extname(path.name)]; + if (!os) return; + return { + name: path.name, + url: `${DOWNLOAD_ENDPOINT + tag}/${path.name}`, + os, + date: new Date(path.date), + }; +}; + +const getFilesForTag = async (tag) => + ( + await s3Ls(S3_BUCKET_PATH + tag + "/").then((listing) => { + log("Calculating hashes for files"); + return (listing || "").split("\n").map(parseS3Listing(tag)); + }) + ).filter((file) => file); + +const calculateFileChecksums = async (files) => + Promise.all( + files.map(async (file) => { + const checksum = await s3Hash(file.url); + log("The hash for", file.name, "is", checksum); + return { + ...file, + checksum, + }; + }) + ); + +// Generates the descriptor for a given tag +// If no tag is provided, it will get the latest tag +// An example descriptor: +// +// { +// "version": "0.1.0", +// "tag": "v0.1.0-new-setup", +// "released": "2023-02-30T12:53:59.412Z", +// "files": [ +// { +// "name": "StremioSetup.exe", +// "url": "https://s3-eu-west-1.amazonaws.com/stremio-artifacts/stremio-shell-ng/v0.1.0-new-setup/StremioSetup.exe", +// "checksum": "0ff94905df4d94233d14f48ed68e31664a478a29204be4c7867c2389929c6ac3", +// "os": "windows" +// } +// ] +// } + +const generateDescriptor = async (args) => { + let tag = args.tag; + if (!tag) { + log("Obtaining the latest tag"); + tag = await s3Ls(S3_BUCKET_PATH).then((listing) => { + // get the first line, remove the DIR prefix, and get the basename + // which is the tag + const first_path = listing.replace(/^\s+\w+\s+/gm, '').split('\n').find(line => line.match(VERSION_REGEX)); + return posix.basename(first_path); + }); + } + const desc_name = tag + ".json"; + const s3_rc_desc_path = S3_VERSIONS_RC_PATH + desc_name; + const s3_dest_path = args.release + ? S3_VERSIONS_PATH + desc_name + : s3_rc_desc_path; + if (!args.force && await s3Ls(s3_dest_path).then((listing) => !!listing)) { + throw new Error( + `${args.release ? "" : "RC "}Descriptor for tag ${tag} already exists` + ); + } + if ( + args.release && + !args.force && + (await s3Ls(s3_rc_desc_path).then((listing) => !!listing)) + ) { + log( + "Descriptor for tag", + tag, + "already exists in the RC folder. Moving it to the releases folder" + ); + if (!args.dry_run) await s3Cp(s3_rc_desc_path, s3_dest_path); + log("Done"); + return; + } + + log("Getting files for tag", tag); + if (!tag) throw new Error("No tag found"); + const version = (tag.match(VERSION_REGEX) || [])[1]; + if (!version) throw new Error("No valid version found"); + const file_listing = await getFilesForTag(tag); + // We need at least one file to extract the release date + if (!file_listing.length) throw new Error("No files found"); + if (args.wait_all && file_listing.length < Object.keys(OS_EXT).length) { + log( + `Not all files are uploaded yet. Rerun this script after ${Object.keys(OS_EXT).length - file_listing.length + } more are uploaded` + ); + return; + } + const files = await calculateFileChecksums(file_listing); + const descriptor = { + version, + tag, + released: file_listing[0].date.toISOString(), + files, + }; + const descriptor_text = JSON.stringify(descriptor, null, 2) + "\n"; + if (args.dry_run) { + process.stdout.write(descriptor_text); + return; + } + + const descriptor_path = `${tmpdir()}/${desc_name}`; + log("Writting descriptor to", descriptor_path); + fs.writeFileSync(descriptor_path, descriptor_text); + + log(`Uploading ${args.release ? "" : "RC "}descriptor to S3`); + try { + await s3Cp(descriptor_path, s3_dest_path); + } finally { + // Clean up the temporary file even if the upload fails + log("Cleaning up"); + fs.unlinkSync(descriptor_path); + } + log("Done"); +}; + +generateDescriptor(args).catch((err) => { + console.error(err.message); + process.exit(1); +});