From e2b19e5b3d1b6b8314efb5353abbc86d4b9027f5 Mon Sep 17 00:00:00 2001 From: klzgrad Date: Sat, 5 Oct 2019 15:52:34 +0800 Subject: [PATCH] build: Add sysroot creator script --- .../linux/sysroot_scripts/generate_keyring.sh | 8 +- .../linux/sysroot_scripts/install-sysroot.py | 2 +- .../linux/sysroot_scripts/sysroot_creator.py | 512 +----------------- src/build/linux/sysroot_scripts/sysroots.json | 6 + 4 files changed, 31 insertions(+), 497 deletions(-) diff --git a/src/build/linux/sysroot_scripts/generate_keyring.sh b/src/build/linux/sysroot_scripts/generate_keyring.sh index 87cecab5df..6cd9fcfb1e 100755 --- a/src/build/linux/sysroot_scripts/generate_keyring.sh +++ b/src/build/linux/sysroot_scripts/generate_keyring.sh @@ -9,8 +9,14 @@ set -o errexit SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" KEYS=( + # Debian Ports Archive Automatic Signing Key (2023) + "B523E5F3FC4E5F2C" + # Debian Ports Archive Automatic Signing Key (2024) + "8D69674688B6CB36" # Debian Archive Automatic Signing Key (12/bookworm) - "6ED0E7B82643E131" + "B7C5D7D6350947F8" + # Debian Security Archive Automatic Signing Key (12/bookworm) + "254CF3B5AEC0A8F0" # Debian Stable Release Key (12/bookworm) "F8D2585B8783D481" # Debian Archive Automatic Signing Key (11/bullseye) diff --git a/src/build/linux/sysroot_scripts/install-sysroot.py b/src/build/linux/sysroot_scripts/install-sysroot.py index abd58136ef..9113d85f19 100755 --- a/src/build/linux/sysroot_scripts/install-sysroot.py +++ b/src/build/linux/sysroot_scripts/install-sysroot.py @@ -36,7 +36,7 @@ from urllib.request import urlopen SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(SCRIPT_DIR))) -VALID_ARCHS = ("amd64", "i386", "armhf", "arm64", "mipsel", "mips64el") +VALID_ARCHS = ('amd64', 'i386', 'armhf', 'arm64', 'mipsel', 'mips64el', 'riscv64') ARCH_TRANSLATIONS = { "x64": "amd64", diff --git a/src/build/linux/sysroot_scripts/sysroot_creator.py b/src/build/linux/sysroot_scripts/sysroot_creator.py index 6ada8afb20..deb79a3319 100755 --- a/src/build/linux/sysroot_scripts/sysroot_creator.py +++ b/src/build/linux/sysroot_scripts/sysroot_creator.py @@ -39,10 +39,6 @@ ARCHIVE_TIMESTAMP = "20230611T210420Z" ARCHIVE_URL = f"https://snapshot.debian.org/archive/debian/{ARCHIVE_TIMESTAMP}/" APT_SOURCES_LIST = [ - # Debian 12 (Bookworm) is needed for GTK4. It should be kept before - # bullseye so that bullseye takes precedence. - ("bookworm", ["main"]), - ("bookworm-updates", ["main"]), # This mimics a sources.list from bullseye. ("bullseye", ["main", "contrib", "non-free"]), ("bullseye-updates", ["main", "contrib", "non-free"]), @@ -56,6 +52,7 @@ TRIPLES = { "arm64": "aarch64-linux-gnu", "mipsel": "mipsel-linux-gnu", "mips64el": "mips64el-linux-gnuabi64", + "riscv64": "riscv64-linux-gnu", } REQUIRED_TOOLS = [ @@ -74,520 +71,48 @@ RELEASE_FILE_GPG = "Release.gpg" # Packages common to all architectures. DEBIAN_PACKAGES = [ - "comerr-dev", - "krb5-multidev", - "libaom0", - "libaom3", - "libasound2", - "libasound2-dev", - "libasyncns0", - "libatk-bridge2.0-0", - "libatk-bridge2.0-dev", - "libatk1.0-0", - "libatk1.0-dev", "libatomic1", - "libatspi2.0-0", - "libatspi2.0-dev", - "libattr1", - "libaudit1", - "libavahi-client3", - "libavahi-common3", - "libavcodec-dev", - "libavcodec58", - "libavcodec59", - "libavformat-dev", - "libavformat59", - "libavutil-dev", - "libavutil56", - "libavutil57", - "libb2-1", - "libblkid-dev", - "libblkid1", - "libbluetooth-dev", - "libbluetooth3", - "libbluray2", - "libbrotli-dev", - "libbrotli1", - "libbsd0", - "libbz2-1.0", "libc6", "libc6-dev", - "libcairo-gobject2", - "libcairo-script-interpreter2", - "libcairo2", - "libcairo2-dev", - "libcap-dev", - "libcap-ng0", - "libcap2", - "libchromaprint1", - "libcjson1", - "libcloudproviders0", - "libcodec2-0.9", - "libcodec2-1.0", - "libcolord2", - "libcom-err2", - "libcrypt-dev", "libcrypt1", - "libcups2", - "libcups2-dev", - "libcupsimage2", - "libcupsimage2-dev", - "libcurl3-gnutls", - "libcurl4-gnutls-dev", - "libdatrie-dev", - "libdatrie1", - "libdav1d4", - "libdav1d6", - "libdb5.3", - "libdbus-1-3", - "libdbus-1-dev", - "libdbus-glib-1-2", - "libdbusmenu-glib-dev", - "libdbusmenu-glib4", - "libdbusmenu-gtk3-4", - "libdbusmenu-gtk4", - "libdeflate-dev", - "libdeflate0", - "libdouble-conversion3", - "libdrm-amdgpu1", - "libdrm-dev", - "libdrm-nouveau2", - "libdrm-radeon1", - "libdrm2", - "libegl-dev", - "libegl1", - "libegl1-mesa", - "libegl1-mesa-dev", - "libelf-dev", - "libelf1", - "libepoxy-dev", - "libepoxy0", - "libevdev-dev", - "libevdev2", - "libevent-2.1-7", - "libexpat1", - "libexpat1-dev", - "libffi-dev", - "libffi7", - "libflac-dev", - "libflac8", - "libfontconfig-dev", - "libfontconfig1", - "libfreetype-dev", - "libfreetype6", - "libfribidi-dev", - "libfribidi0", - "libgbm-dev", - "libgbm1", "libgcc-10-dev", "libgcc-s1", - "libgcrypt20", - "libgcrypt20-dev", - "libgdk-pixbuf-2.0-0", - "libgdk-pixbuf-2.0-dev", - "libgl-dev", - "libgl1", - "libgl1-mesa-dev", - "libgl1-mesa-glx", - "libglapi-mesa", - "libgles-dev", - "libgles1", - "libgles2", - "libglib2.0-0", - "libglib2.0-dev", - "libglvnd-dev", - "libglvnd0", - "libglx-dev", - "libglx0", - "libgme0", - "libgmp10", - "libgnutls-dane0", - "libgnutls-openssl27", - "libgnutls28-dev", - "libgnutls30", - "libgnutlsxx28", "libgomp1", - "libgpg-error-dev", - "libgpg-error0", - "libgraphene-1.0-0", - "libgraphene-1.0-dev", - "libgraphite2-3", - "libgraphite2-dev", - "libgsm1", - "libgssapi-krb5-2", - "libgssrpc4", - "libgtk-3-0", - "libgtk-3-dev", - "libgtk-4-1", - "libgtk-4-dev", - "libgtk2.0-0", - "libgudev-1.0-0", - "libharfbuzz-dev", - "libharfbuzz-gobject0", - "libharfbuzz-icu0", - "libharfbuzz0b", - "libhogweed6", - "libhwy1", - "libice6", - "libicu-le-hb0", - "libicu67", - "libidl-2-0", - "libidn11", - "libidn2-0", - "libinput-dev", - "libinput10", - "libjbig-dev", - "libjbig0", - "libjpeg62-turbo", - "libjpeg62-turbo-dev", - "libjson-glib-1.0-0", - "libjsoncpp-dev", - "libjsoncpp24", - "libjxl0.7", - "libjxl0.7", - "libk5crypto3", - "libkadm5clnt-mit12", - "libkadm5srv-mit12", - "libkdb5-10", - "libkeyutils1", - "libkrb5-3", - "libkrb5-dev", - "libkrb5support0", - "liblcms2-2", - "libldap-2.4-2", - "liblerc4", - "libltdl7", - "liblz4-1", - "liblzma5", - "liblzo2-2", - "libmbedcrypto7", - "libmd0", - "libmd4c0", - "libminizip-dev", - "libminizip1", - "libmount-dev", - "libmount1", - "libmp3lame0", - "libmpg123-0", - "libmtdev1", - "libncurses-dev", - "libncurses6", - "libncursesw6", - "libnettle8", - "libnghttp2-14", - "libnorm1", - "libnsl2", - "libnspr4", - "libnspr4-dev", - "libnss-db", - "libnss3", - "libnss3-dev", - "libnuma1", - "libogg-dev", - "libogg0", - "libopengl0", - "libopenjp2-7", - "libopenmpt0", - "libopus-dev", - "libopus-dev", - "libopus0", - "libp11-kit0", - "libpam0g", - "libpam0g-dev", - "libpango-1.0-0", - "libpango1.0-dev", - "libpangocairo-1.0-0", - "libpangoft2-1.0-0", - "libpangox-1.0-0", - "libpangoxft-1.0-0", - "libpci-dev", - "libpci3", - "libpciaccess0", - "libpcre16-3", - "libpcre2-16-0", - "libpcre2-32-0", - "libpcre2-8-0", - "libpcre2-dev", - "libpcre2-posix2", - "libpcre3", - "libpcre3-dev", - "libpcre32-3", - "libpcrecpp0v5", - "libpgm-5.3-0", - "libpipewire-0.3-0", - "libpipewire-0.3-dev", - "libpixman-1-0", - "libpixman-1-dev", - "libpng-dev", - "libpng16-16", - "libproxy1v5", - "libpsl5", - "libpthread-stubs0-dev", - "libpulse-dev", - "libpulse-mainloop-glib0", - "libpulse0", - "libqt5concurrent5", - "libqt5core5a", - "libqt5dbus5", - "libqt5gui5", - "libqt5network5", - "libqt5printsupport5", - "libqt5sql5", - "libqt5test5", - "libqt5widgets5", - "libqt5xml5", - "libqt6concurrent6", - "libqt6core6", - "libqt6dbus6", - "libqt6gui6", - "libqt6network6", - "libqt6opengl6", - "libqt6openglwidgets6", - "libqt6printsupport6", - "libqt6sql6", - "libqt6test6", - "libqt6widgets6", - "libqt6xml6", - "librabbitmq4", - "librav1e0", - "libre2-9", - "libre2-dev", - "librest-0.7-0", - "librist4", - "librsvg2-2", - "librtmp1", - "libsasl2-2", - "libselinux1", - "libselinux1-dev", - "libsepol1", - "libsepol1-dev", - "libshine3", - "libsm6", - "libsnappy-dev", - "libsnappy1v5", - "libsndfile1", - "libsodium23", - "libsoup-gnome2.4-1", - "libsoup2.4-1", - "libsoxr0", - "libspa-0.2-dev", - "libspeechd-dev", - "libspeechd2", - "libspeex1", - "libsqlite3-0", - "libsrt1.5-gnutls", - "libssh-gcrypt-4", - "libssh2-1", - "libssl-dev", - "libssl1.1", "libstdc++-10-dev", "libstdc++6", - "libsvtav1enc1", - "libswresample-dev", - "libswresample3", - "libswresample4", - "libsystemd-dev", - "libsystemd0", - "libtasn1-6", - "libthai-dev", - "libthai0", - "libtheora0", - "libtheora0", - "libtiff-dev", - "libtiff5", - "libtiff6", - "libtiffxx5", - "libtinfo6", - "libtirpc3", - "libts0", - "libtwolame0", - "libudev-dev", - "libudev1", - "libudfread0", - "libunbound8", - "libunistring2", - "libutempter-dev", - "libutempter0", - "libuuid1", - "libva-dev", - "libva-drm2", - "libva-glx2", - "libva-wayland2", - "libva-x11-2", - "libva2", - "libvdpau1", - "libvorbis0a", - "libvorbisenc2", - "libvorbisfile3", - "libvpx-dev", - "libvpx6", - "libvpx7", - "libvulkan-dev", - "libvulkan1", - "libwacom2", - "libwavpack1", - "libwayland-bin", - "libwayland-client0", - "libwayland-cursor0", - "libwayland-dev", - "libwayland-egl-backend-dev", - "libwayland-egl1", - "libwayland-egl1-mesa", - "libwayland-server0", - "libwebp-dev", - "libwebp6", - "libwebp7", - "libwebpdemux2", - "libwebpmux3", - "libwrap0", - "libx11-6", - "libx11-dev", - "libx11-xcb-dev", - "libx11-xcb1", - "libx264-160", - "libx264-164", - "libx265-192", - "libx265-199", - "libxau-dev", - "libxau6", - "libxcb-dri2-0", - "libxcb-dri2-0-dev", - "libxcb-dri3-0", - "libxcb-dri3-dev", - "libxcb-glx0", - "libxcb-glx0-dev", - "libxcb-icccm4", - "libxcb-image0", - "libxcb-image0-dev", - "libxcb-keysyms1", - "libxcb-present-dev", - "libxcb-present0", - "libxcb-randr0", - "libxcb-randr0-dev", - "libxcb-render-util0", - "libxcb-render-util0-dev", - "libxcb-render0", - "libxcb-render0-dev", - "libxcb-shape0", - "libxcb-shape0-dev", - "libxcb-shm0", - "libxcb-shm0-dev", - "libxcb-sync-dev", - "libxcb-sync1", - "libxcb-util-dev", - "libxcb-util1", - "libxcb-xfixes0", - "libxcb-xfixes0-dev", - "libxcb-xinerama0", - "libxcb-xinput0", - "libxcb-xkb1", - "libxcb1", - "libxcb1-dev", - "libxcomposite-dev", - "libxcomposite1", - "libxcursor-dev", - "libxcursor1", - "libxdamage-dev", - "libxdamage1", - "libxdmcp-dev", - "libxdmcp6", - "libxext-dev", - "libxext6", - "libxfixes-dev", - "libxfixes3", - "libxft-dev", - "libxft2", - "libxi-dev", - "libxi6", - "libxinerama-dev", - "libxinerama1", - "libxkbcommon-dev", - "libxkbcommon-x11-0", - "libxkbcommon0", - "libxml2", - "libxml2-dev", - "libxrandr-dev", - "libxrandr2", - "libxrender-dev", - "libxrender1", - "libxshmfence-dev", - "libxshmfence1", - "libxslt1-dev", - "libxslt1.1", - "libxss-dev", - "libxss1", - "libxt-dev", - "libxt6", - "libxtst-dev", - "libxtst6", - "libxvidcore4", - "libxxf86vm-dev", - "libxxf86vm1", - "libzmq5", - "libzstd1", - "libzvbi0", "linux-libc-dev", - "mesa-common-dev", - "ocl-icd-libopencl1", - "qt6-base-dev", - "qt6-base-dev-tools", - "qtbase5-dev", - "qtbase5-dev-tools", - "shared-mime-info", - "uuid-dev", - "wayland-protocols", - "x11proto-dev", - "zlib1g", - "zlib1g-dev", ] DEBIAN_PACKAGES_ARCH = { "amd64": [ "libasan6", - "libdrm-intel1", "libitm1", "liblsan0", - "libmfx1", "libquadmath0", "libtsan0", "libubsan1", - "valgrind", ], "i386": [ "libasan6", - "libdrm-intel1", "libitm1", "libquadmath0", "libubsan1", - "valgrind", ], "armhf": [ "libasan6", - "libdrm-etnaviv1", - "libdrm-exynos1", - "libdrm-freedreno1", - "libdrm-omap1", - "libdrm-tegra0", "libubsan1", - "valgrind", ], "arm64": [ "libasan6", - "libdrm-etnaviv1", - "libdrm-freedreno1", - "libdrm-tegra0", "libgmp10", "libitm1", "liblsan0", - "libthai0", "libtsan0", "libubsan1", - "valgrind", ], "mipsel": [], - "mips64el": [ - "valgrind", - ], + "mips64el": [], + "riscv64": [], } @@ -729,11 +254,13 @@ def generate_package_list_dist_repo(arch: str, dist: str, download_or_copy_non_unique_filename(package_list_arch, package_list) verify_package_listing(package_file_arch, package_list, dist) + # `not line.endswith(":")` is added here to handle the case of + # "X-Cargo-Built-Using:\n rust-adler (= 1.0.2-2), ..." with lzma.open(package_list, "rt") as src: return [ dict( line.split(": ", 1) for line in package_meta.splitlines() - if not line.startswith(" ")) + if not line.startswith(" ") and not line.endswith(":")) for package_meta in src.read().split("\n\n") if package_meta ] @@ -775,13 +302,6 @@ def hacks_and_patches(install_root: str, script_dir: str, arch: str) -> None: if os.path.exists(qtchooser_conf): os.remove(qtchooser_conf) - # libxcomposite1 is missing a symbols file. - atomic_copyfile( - os.path.join(script_dir, "libxcomposite1-symbols"), - os.path.join(install_root, "debian", "libxcomposite1", "DEBIAN", - "symbols"), - ) - # __GLIBC_MINOR__ is used as a feature test macro. Replace it with the # earliest supported version of glibc (2.26). features_h = os.path.join(install_root, "usr", "include", "features.h") @@ -827,16 +347,13 @@ def hacks_and_patches(install_root: str, script_dir: str, arch: str) -> None: # Avoid requiring unsupported glibc versions. for lib in ["libc.so.6", "libm.so.6", "libcrypt.so.1"]: + # RISCV64 is new and has no backward compatibility. + # Reversioning would remove necessary symbols and cause linking failures. + if arch == "riscv64": + continue lib_path = os.path.join(install_root, "lib", TRIPLES[arch], lib) reversion_glibc.reversion_glibc(lib_path) - # GTK4 is provided by bookworm (12), but pango is provided by bullseye - # (11). Fix the GTK4 pkgconfig file to relax the pango version - # requirement. - gtk4_pc = os.path.join(pkgconfig_dir, "gtk4.pc") - replace_in_file(gtk4_pc, r"pango [>=0-9. ]*", "pango") - replace_in_file(gtk4_pc, r"pangocairo [>=0-9. ]*", "pangocairo") - def replace_in_file(file_path: str, search_pattern: str, replace_pattern: str) -> None: @@ -944,7 +461,6 @@ def cleanup_jail_symlinks(install_root: str) -> None: os.remove(full_path) os.symlink(relative_path, full_path) - def verify_library_deps(install_root: str) -> None: """ Verifies if all required libraries are present in the sysroot environment. @@ -971,6 +487,7 @@ def verify_library_deps(install_root: str) -> None: output = subprocess.check_output(cmd_readelf).decode() for line in output.split("\n"): if "NEEDED" in line: + print(file, line) needed_libs.add(line.split("[")[1].split("]")[0]) missing_libs = needed_libs - shared_libs @@ -986,7 +503,6 @@ def build_sysroot(arch: str) -> None: hacks_and_patches(install_root, SCRIPT_DIR, arch) cleanup_jail_symlinks(install_root) verify_library_deps(install_root) - create_tarball(install_root, arch) def upload_sysroot(arch: str) -> str: @@ -1053,6 +569,12 @@ def main(): sanity_check() + # RISCV64 only has support in debian-ports, no support in bookworm. + if args.architecture == "riscv64": + global ARCHIVE_URL, APT_SOURCES_LIST + ARCHIVE_URL = "https://snapshot.debian.org/archive/debian-ports/20230724T141507Z/" + APT_SOURCES_LIST = [("sid", ["main"])] + if args.command == "build": build_sysroot(args.architecture) elif args.command == "upload": diff --git a/src/build/linux/sysroot_scripts/sysroots.json b/src/build/linux/sysroot_scripts/sysroots.json index f146ea68bc..61811dac19 100644 --- a/src/build/linux/sysroot_scripts/sysroots.json +++ b/src/build/linux/sysroot_scripts/sysroots.json @@ -34,5 +34,11 @@ "SysrootDir": "debian_bullseye_mipsel-sysroot", "Tarball": "debian_bullseye_mipsel_sysroot.tar.xz", "URL": "https://commondatastorage.googleapis.com/chrome-linux-sysroot" + }, + "bullseye_riscv64": { + "Key": "", + "Sha1Sum": "", + "SysrootDir": "debian_bullseye_riscv64-sysroot", + "Tarball": "debian_bullseye_riscv64_sysroot.tar.xz" } }